diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 590d7834b2b8be..9f808af38e69df 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -6,7 +6,7 @@ ENV WASI_SDK_VERSION=20 ENV WASI_SDK_PATH=/opt/wasi-sdk ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=9.0.1 +ENV WASMTIME_VERSION=14.0.4 ENV WASMTIME_CPU_ARCH=x86_64 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ diff --git a/.gitattributes b/.gitattributes index 8c37dbbb631022..2a48df079e1aeb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -76,6 +76,7 @@ Include/internal/pycore_ast_state.h generated Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated +Include/internal/pycore_uop_ids.h generated Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated @@ -84,6 +85,7 @@ Lib/keyword.py generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated Lib/token.py generated +Misc/sbom.spdx.json generated Objects/typeslots.inc generated PC/python3dll.c generated Parser/parser.c generated @@ -92,7 +94,6 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Python/abstract_interp_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1925363cbeb46e..8038206441ab9b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -40,6 +40,7 @@ Lib/test/test_patma.py @brandtbucher Lib/test/test_peepholer.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon @gvanrossum +Tools/c-analyzer/ @ericsnowcurrently # Exceptions Lib/traceback.py @iritkatriel @@ -156,6 +157,8 @@ Doc/c-api/stable.rst @encukou **/*dataclasses* @ericvsmith +**/*ensurepip* @pfmoore @pradyunsg + **/*idlelib* @terryjreedy **/*typing* @JelleZijlstra @AlexWaygood @@ -188,5 +191,14 @@ Doc/c-api/stable.rst @encukou /Lib/test/test_clinic.py @erlend-aasland @AlexWaygood Doc/howto/clinic.rst @erlend-aasland +# Subinterpreters +Lib/test/support/interpreters/ @ericsnowcurrently +Modules/_xx*interp*module.c @ericsnowcurrently +Lib/test/test_interpreters/ @ericsnowcurrently + # WebAssembly /Tools/wasm/ @brettcannon + +# SBOM +/Misc/sbom.spdx.json @sethmlarson +/Tools/build/generate_sbom.py @sethmlarson diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 1ef9178b95e5f6..570b8779994a0f 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -19,7 +19,7 @@ jobs: permissions: issues: write steps: - - uses: actions/github-script@v6 + - uses: actions/github-script@v7 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4210194b978749..2168ec101cf3d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -128,7 +128,7 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - name: Restore config.cache @@ -187,13 +187,13 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-windows.yml - build_windows_free_threaded: - name: 'Windows (free-threaded)' + build_windows_free_threading: + name: 'Windows (free-threading)' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'topic-free-threaded') + if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-windows.yml with: - free-threaded: true + free-threading: true build_macos: name: 'macOS' @@ -203,14 +203,14 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} - build_macos_free_threaded: - name: 'macOS (free-threaded)' + build_macos_free_threading: + name: 'macOS (free-threading)' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'topic-free-threaded') + if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - free-threaded: true + free-threading: true build_ubuntu: name: 'Ubuntu' @@ -225,10 +225,10 @@ jobs: --with-pydebug \ --with-openssl=$OPENSSL_DIR - build_ubuntu_free_threaded: - name: 'Ubuntu (free-threaded)' + build_ubuntu_free_threading: + name: 'Ubuntu (free-threading)' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'topic-free-threaded') + if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-ubuntu.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} @@ -395,7 +395,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: hypothesis-example-db @@ -483,14 +483,14 @@ jobs: output-sarif: true sanitizer: ${{ matrix.sanitizer }} - name: Upload crash - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() && steps.build.outcome == 'success' with: name: ${{ matrix.sanitizer }}-artifacts path: ./out/artifacts - name: Upload SARIF if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: cifuzz-sarif/results.sarif checkout_path: cifuzz-sarif @@ -504,12 +504,12 @@ jobs: - check-docs - check_generated_files - build_macos - - build_macos_free_threaded + - build_macos_free_threading - build_ubuntu - - build_ubuntu_free_threaded + - build_ubuntu_free_threading - build_ubuntu_ssltests - build_windows - - build_windows_free_threaded + - build_windows_free_threading - test_hypothesis - build_asan - cifuzz @@ -521,10 +521,7 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- - build_macos_free_threaded, - build_ubuntu_free_threaded, build_ubuntu_ssltests, - build_windows_free_threaded, cifuzz, test_hypothesis, allowed-skips: >- @@ -540,12 +537,12 @@ jobs: && ' check_generated_files, build_macos, - build_macos_free_threaded, + build_macos_free_threading, build_ubuntu, - build_ubuntu_free_threaded, + build_ubuntu_free_threading, build_ubuntu_ssltests, build_windows, - build_windows_free_threaded, + build_windows_free_threading, build_asan, ' || '' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6c1c29a58cf4fc..4a70ec6205a05b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 405511ca6820b3..11928e72b9b43a 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,6 +8,8 @@ on: pull_request: paths: - ".github/workflows/mypy.yml" + - "Lib/test/libregrtest/**" + - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" - "Tools/clinic/**" - "Tools/peg_generator/**" @@ -32,6 +34,8 @@ jobs: strategy: matrix: target: [ + "Lib/test/libregrtest", + "Tools/build/", "Tools/cases_generator", "Tools/clinic", "Tools/peg_generator", @@ -42,7 +46,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" cache: pip diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 4599b21ef35f05..9f1a8a824e5f19 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -18,7 +18,7 @@ jobs: node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v6 + uses: actions/github-script@v7 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index bbae378f7b994e..0800401f4cd113 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -21,6 +21,7 @@ apt-get -yq install \ libssl-dev \ lzma \ lzma-dev \ + strace \ tk-dev \ uuid-dev \ xvfb \ diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 1c4fa4239c1e34..e534751ee1011d 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -41,7 +41,7 @@ jobs: git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3' cache: 'pip' @@ -72,7 +72,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: 'Set up Python' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' # known to work with Sphinx 4.2 cache: 'pip' diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 22f46d18e1b43a..c24b6e963ddfd6 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -4,7 +4,7 @@ on: config_hash: required: true type: string - free-threaded: + free-threading: required: false type: boolean default: false @@ -35,7 +35,7 @@ jobs: ./configure \ --config-cache \ --with-pydebug \ - ${{ inputs.free-threaded && '--disable-gil' || '' }} \ + ${{ inputs.free-threading && '--disable-gil' || '' }} \ --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 29e0a7e35b5450..ae27c108d8368c 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -1,7 +1,7 @@ on: workflow_call: inputs: - free-threaded: + free-threading: required: false type: boolean default: false @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build CPython - run: .\PCbuild\build.bat -e -d -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p Win32 ${{ inputs.free-threading && '--disable-gil' || '' }} - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests @@ -33,7 +33,7 @@ jobs: - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p x64 ${{ inputs.free-threading && '--disable-gil' || '' }} - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests @@ -50,4 +50,4 @@ jobs: - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }} + run: .\PCbuild\build.bat -e -d -v -p arm64 ${{ inputs.free-threading && '--disable-gil' || '' }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 94676f5ee5fffc..07608fe91b4dbe 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@v8 + uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 4a545037bf6e2b..83b007f1c9c2ef 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -26,7 +26,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.gitignore b/.gitignore index 8c8273fc7a3aa3..c424a894c2a6e0 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,7 @@ Tools/unicode/data/ /config.status.lineno # hendrikmuhs/ccache-action@v1 /.ccache +/cross-build/ /platform /profile-clean-stamp /profile-run-stamp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 35d9c64a8c5c15..9bd9c59a1ddc74 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.2 + rev: v0.1.7 hooks: - id: ruff name: Run Ruff on Lib/test/ @@ -24,7 +24,7 @@ repos: types_or: [c, inc, python, rst] - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.8.1 + rev: v0.9.1 hooks: - id: sphinx-lint args: [--enable=default-role] diff --git a/Doc/Makefile b/Doc/Makefile index 7af56e965e1be4..38fd60f2ae01d1 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -19,8 +19,12 @@ SPHINXERRORHANDLING = -W PAPEROPT_a4 = -D latex_elements.papersize=a4paper PAPEROPT_letter = -D latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j $(JOBS) \ - $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) +ALLSPHINXOPTS = -b $(BUILDER) \ + -d build/doctrees \ + -j $(JOBS) \ + $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ + . build/$(BUILDER) $(SOURCES) .PHONY: help help: @@ -142,7 +146,7 @@ htmlview: html .PHONY: htmllive htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild -htmllive: SPHINXOPTS = --re-ignore="/venv/" +htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 htmllive: html .PHONY: clean diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 62d87d898e682c..834aae9372fe3b 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -413,7 +413,7 @@ API Functions than a variable number of arguments. -.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...) +.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char * const *keywords, ...) Parse the parameters of a function that takes both positional and keyword parameters into local variables. @@ -424,15 +424,24 @@ API Functions Returns true on success; on failure, it returns false and raises the appropriate exception. + .. note:: + + The *keywords* parameter declaration is :c:expr:`char * const *` in C and + :c:expr:`const char * const *` in C++. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + .. versionchanged:: 3.6 Added support for :ref:`positional-only parameters `. .. versionchanged:: 3.13 + The *keywords* parameter has now type :c:expr:`char * const *` in C and + :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. Added support for non-ASCII keyword parameter names. -.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs) + +.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char * const *keywords, va_list vargs) Identical to :c:func:`PyArg_ParseTupleAndKeywords`, except that it accepts a va_list rather than a variable number of arguments. @@ -505,6 +514,19 @@ API Functions PyArg_ParseTuple(args, "O|O:ref", &object, &callback) +.. c:macro:: PY_CXX_CONST + + The value to be inserted, if any, before :c:expr:`char * const *` + in the *keywords* parameter declaration of + :c:func:`PyArg_ParseTupleAndKeywords` and + :c:func:`PyArg_VaParseTupleAndKeywords`. + Default empty for C and ``const`` for C++ + (:c:expr:`const char * const *`). + To override, define it to the desired value before including + :file:`Python.h`. + + .. versionadded:: 3.13 + --------------- Building values diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index a3a63b38c432f2..c7e3cd9463e5d7 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -440,7 +440,7 @@ Querying the error indicator .. c:function:: PyObject *PyErr_GetRaisedException(void) Return the exception currently being raised, clearing the error indicator at - the same time. + the same time. Return ``NULL`` if the error indicator is not set. This function is used by code that needs to catch exceptions, or code that needs to save and restore the error indicator temporarily. @@ -541,7 +541,8 @@ Querying the error indicator .. note:: - This function *does not* implicitly set the ``__traceback__`` + This function *does not* implicitly set the + :attr:`~BaseException.__traceback__` attribute on the exception value. If setting the traceback appropriately is desired, the following additional snippet is needed:: @@ -753,7 +754,8 @@ Exception Objects .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as - accessible from Python through :attr:`__traceback__`. If there is no + accessible from Python through the :attr:`~BaseException.__traceback__` + attribute. If there is no traceback associated, this returns ``NULL``. @@ -767,8 +769,8 @@ Exception Objects Return the context (another exception instance during whose handling *ex* was raised) associated with the exception as a new reference, as accessible from - Python through :attr:`__context__`. If there is no context associated, this - returns ``NULL``. + Python through the :attr:`~BaseException.__context__` attribute. + If there is no context associated, this returns ``NULL``. .. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) @@ -782,7 +784,8 @@ Exception Objects Return the cause (either an exception instance, or ``None``, set by ``raise ... from ...``) associated with the exception as a new - reference, as accessible from Python through :attr:`__cause__`. + reference, as accessible from Python through the + :attr:`~BaseException.__cause__` attribute. .. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) @@ -791,7 +794,8 @@ Exception Objects it. There is no type check to make sure that *cause* is either an exception instance or ``None``. This steals a reference to *cause*. - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. + The :attr:`~BaseException.__suppress_context__` attribute is implicitly set + to ``True`` by this function. .. c:function:: PyObject* PyException_GetArgs(PyObject *ex) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1accee2767a485..6bb1e9b5803b58 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -50,7 +50,7 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) - Get the *frame*'s ``f_builtins`` attribute. + Get the *frame*'s :attr:`~frame.f_builtins` attribute. Return a :term:`strong reference`. The result cannot be ``NULL``. @@ -81,7 +81,7 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetGlobals(PyFrameObject *frame) - Get the *frame*'s ``f_globals`` attribute. + Get the *frame*'s :attr:`~frame.f_globals` attribute. Return a :term:`strong reference`. The result cannot be ``NULL``. @@ -90,7 +90,7 @@ See also :ref:`Reflection `. .. c:function:: int PyFrame_GetLasti(PyFrameObject *frame) - Get the *frame*'s ``f_lasti`` attribute. + Get the *frame*'s :attr:`~frame.f_lasti` attribute. Returns -1 if ``frame.f_lasti`` is ``None``. @@ -120,7 +120,7 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) - Get the *frame*'s ``f_locals`` attribute (:class:`dict`). + Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`). Return a :term:`strong reference`. diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 5857dba82c11c6..e7fb5090c09933 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -34,18 +34,20 @@ There are a few functions specific to Python functions. Return a new function object associated with the code object *code*. *globals* must be a dictionary with the global variables accessible to the function. - The function's docstring and name are retrieved from the code object. *__module__* + The function's docstring and name are retrieved from the code object. + :attr:`~function.__module__` is retrieved from *globals*. The argument defaults, annotations and closure are - set to ``NULL``. *__qualname__* is set to the same value as the code object's - ``co_qualname`` field. + set to ``NULL``. :attr:`~function.__qualname__` is set to the same value as + the code object's :attr:`~codeobject.co_qualname` field. .. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) As :c:func:`PyFunction_New`, but also allows setting the function object's - ``__qualname__`` attribute. *qualname* should be a unicode object or ``NULL``; - if ``NULL``, the ``__qualname__`` attribute is set to the same value as the - code object's ``co_qualname`` field. + :attr:`~function.__qualname__` attribute. + *qualname* should be a unicode object or ``NULL``; + if ``NULL``, the :attr:`!__qualname__` attribute is set to the same value as + the code object's :attr:`~codeobject.co_qualname` field. .. versionadded:: 3.3 @@ -62,11 +64,12 @@ There are a few functions specific to Python functions. .. c:function:: PyObject* PyFunction_GetModule(PyObject *op) - Return a :term:`borrowed reference` to the *__module__* attribute of the - function object *op*. It can be *NULL*. + Return a :term:`borrowed reference` to the :attr:`~function.__module__` + attribute of the :ref:`function object ` *op*. + It can be *NULL*. - This is normally a string containing the module name, but can be set to any - other object by Python code. + This is normally a :class:`string ` containing the module name, + but can be set to any other object by Python code. .. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index 4dc121d7fbaa9b..91d88ae27bc9f4 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -45,4 +45,17 @@ See also the :c:member:`PyTypeObject.tp_hash` member. Get the hash function definition. + .. seealso:: + :pep:`456` "Secure and interchangeable hash algorithm". + .. versionadded:: 3.4 + + +.. c:function:: Py_hash_t Py_HashPointer(const void *ptr) + + Hash a pointer value: process the pointer value as an integer (cast it to + ``uintptr_t`` internally). The pointer is not dereferenced. + + The function cannot fail: it cannot return ``-1``. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 137780cc359cf9..51c20b202f091c 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -154,7 +154,7 @@ Importing Modules :class:`~importlib.machinery.SourceFileLoader` otherwise. The module's :attr:`__file__` attribute will be set to the code object's - :attr:`!co_filename`. If applicable, :attr:`__cached__` will also + :attr:`~codeobject.co_filename`. If applicable, :attr:`__cached__` will also be set. This function will reload the module if it was already imported. See diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index e89641f74c7491..f8fd48e781d6da 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1662,7 +1662,8 @@ Python-level trace functions in previous versions. The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`f_trace_lines` to *0* on that frame. + It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to + *0* on that frame. .. c:var:: int PyTrace_RETURN @@ -1694,7 +1695,7 @@ Python-level trace functions in previous versions. The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not profiling functions) when a new opcode is about to be executed. This event is not emitted by default: it must be explicitly requested by setting - :attr:`f_trace_opcodes` to *1* on the frame. + :attr:`~frame.f_trace_opcodes` to *1* on the frame. .. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 68119a27b18ec2..75e1d46474f1e7 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -34,6 +34,9 @@ of Python objects. Set the object *o* reference counter to *refcnt*. + On :ref:`Python build with Free Threading `, if + *refcnt* is larger than ``UINT32_MAX``, the object is made :term:`immortal`. + This function has no effect on :term:`immortal` objects. .. versionadded:: 3.9 diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 25cb4ed40f63e7..7d82f7839dfcd7 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -419,15 +419,15 @@ Accessing attributes of extension types The string should be static, no copy is made of it. - .. c:member:: Py_ssize_t offset - - The offset in bytes that the member is located on the type’s object struct. - .. c:member:: int type The type of the member in the C struct. See :ref:`PyMemberDef-types` for the possible values. + .. c:member:: Py_ssize_t offset + + The offset in bytes that the member is located on the type’s object struct. + .. c:member:: int flags Zero or more of the :ref:`PyMemberDef-flags`, combined using bitwise OR. @@ -592,7 +592,7 @@ Macro name C type Python type (*): Zero-terminated, UTF8-encoded C string. With :c:macro:`!Py_T_STRING` the C representation is a pointer; - with :c:macro:`!Py_T_STRING_INLINE` the string is stored directly + with :c:macro:`!Py_T_STRING_INPLACE` the string is stored directly in the structure. (**): String of length 1. Only ASCII is accepted. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 10c05beda7c66f..8a26f237652d12 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -343,13 +343,13 @@ slot typedefs | | :c:type:`PyTypeObject` * | | | | :c:type:`Py_ssize_t` | | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`destructor` | void * | void | +| :c:type:`destructor` | :c:type:`PyObject` * | void | +-----------------------------+-----------------------------+----------------------+ | :c:type:`freefunc` | void * | void | +-----------------------------+-----------------------------+----------------------+ | :c:type:`traverseproc` | .. line-block:: | int | | | | | -| | void * | | +| | :c:type:`PyObject` * | | | | :c:type:`visitproc` | | | | void * | | +-----------------------------+-----------------------------+----------------------+ @@ -426,7 +426,7 @@ slot typedefs | | :c:type:`PyObject` * | | | | :c:type:`Py_buffer` * | | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`inquiry` | void * | int | +| :c:type:`inquiry` | :c:type:`PyObject` * | int | +-----------------------------+-----------------------------+----------------------+ | :c:type:`unaryfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index e654412965a727..5541eaa521803b 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -75,19 +75,19 @@ Python: The following APIs are C macros and static inlined functions for fast checks and access to internal read-only data of Unicode objects: -.. c:function:: int PyUnicode_Check(PyObject *o) +.. c:function:: int PyUnicode_Check(PyObject *obj) - Return true if the object *o* is a Unicode object or an instance of a Unicode + Return true if the object *obj* is a Unicode object or an instance of a Unicode subtype. This function always succeeds. -.. c:function:: int PyUnicode_CheckExact(PyObject *o) +.. c:function:: int PyUnicode_CheckExact(PyObject *obj) - Return true if the object *o* is a Unicode object, but not an instance of a + Return true if the object *obj* is a Unicode object, but not an instance of a subtype. This function always succeeds. -.. c:function:: int PyUnicode_READY(PyObject *o) +.. c:function:: int PyUnicode_READY(PyObject *unicode) Returns ``0``. This API is kept only for backward compatibility. @@ -97,17 +97,17 @@ access to internal read-only data of Unicode objects: This API does nothing since Python 3.12. -.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o) +.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *unicode) - Return the length of the Unicode string, in code points. *o* has to be a + Return the length of the Unicode string, in code points. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 -.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *o) - Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *o) - Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *o) +.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *unicode) + Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *unicode) + Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *unicode) Return a pointer to the canonical representation cast to UCS1, UCS2 or UCS4 integer types for direct character access. No checks are performed if the @@ -129,18 +129,18 @@ access to internal read-only data of Unicode objects: ``PyUnicode_WCHAR_KIND`` has been removed. -.. c:function:: int PyUnicode_KIND(PyObject *o) +.. c:function:: int PyUnicode_KIND(PyObject *unicode) Return one of the PyUnicode kind constants (see above) that indicate how many - bytes per character this Unicode object uses to store its data. *o* has to + bytes per character this Unicode object uses to store its data. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 -.. c:function:: void* PyUnicode_DATA(PyObject *o) +.. c:function:: void* PyUnicode_DATA(PyObject *unicode) - Return a void pointer to the raw Unicode buffer. *o* has to be a Unicode + Return a void pointer to the raw Unicode buffer. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 @@ -168,25 +168,25 @@ access to internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *o, Py_ssize_t index) +.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) - Read a character from a Unicode object *o*, which must be in the "canonical" + Read a character from a Unicode object *unicode*, which must be in the "canonical" representation. This is less efficient than :c:func:`PyUnicode_READ` if you do multiple consecutive reads. .. versionadded:: 3.3 -.. c:function:: Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *o) +.. c:function:: Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *unicode) Return the maximum code point that is suitable for creating another string - based on *o*, which must be in the "canonical" representation. This is + based on *unicode*, which must be in the "canonical" representation. This is always an approximation but more efficient than iterating over the string. .. versionadded:: 3.3 -.. c:function:: int PyUnicode_IsIdentifier(PyObject *o) +.. c:function:: int PyUnicode_IsIdentifier(PyObject *unicode) Return ``1`` if the string is a valid identifier according to the language definition, section :ref:`identifiers`. Return ``0`` otherwise. @@ -358,9 +358,9 @@ APIs: .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *str, Py_ssize_t size) - Create a Unicode object from the char buffer *u*. The bytes will be + Create a Unicode object from the char buffer *str*. The bytes will be interpreted as being UTF-8 encoded. The buffer is copied into the new object. The return value might be a shared object, i.e. modification of the data is @@ -369,16 +369,16 @@ APIs: This function raises :exc:`SystemError` when: * *size* < 0, - * *u* is ``NULL`` and *size* > 0 + * *str* is ``NULL`` and *size* > 0 .. versionchanged:: 3.12 - *u* == ``NULL`` with *size* > 0 is not allowed anymore. + *str* == ``NULL`` with *size* > 0 is not allowed anymore. -.. c:function:: PyObject *PyUnicode_FromString(const char *u) +.. c:function:: PyObject *PyUnicode_FromString(const char *str) Create a Unicode object from a UTF-8 encoded null-terminated char buffer - *u*. + *str*. .. c:function:: PyObject* PyUnicode_FromFormat(const char *format, ...) @@ -646,29 +646,29 @@ APIs: .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_Substring(PyObject *str, Py_ssize_t start, \ +.. c:function:: PyObject* PyUnicode_Substring(PyObject *unicode, Py_ssize_t start, \ Py_ssize_t end) - Return a substring of *str*, from character index *start* (included) to + Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. .. versionadded:: 3.3 -.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *u, Py_UCS4 *buffer, \ +.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *unicode, Py_UCS4 *buffer, \ Py_ssize_t buflen, int copy_null) - Copy the string *u* into a UCS4 buffer, including a null character, if + Copy the string *unicode* into a UCS4 buffer, including a null character, if *copy_null* is set. Returns ``NULL`` and sets an exception on error (in particular, a :exc:`SystemError` if *buflen* is smaller than the length of - *u*). *buffer* is returned on success. + *unicode*). *buffer* is returned on success. .. versionadded:: 3.3 -.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u) +.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *unicode) - Copy the string *u* into a new UCS4 buffer that is allocated using + Copy the string *unicode* into a new UCS4 buffer that is allocated using :c:func:`PyMem_Malloc`. If this fails, ``NULL`` is returned with a :exc:`MemoryError` set. The returned buffer always has an extra null code point appended. @@ -683,7 +683,7 @@ The current locale encoding can be used to decode text from the operating system. .. c:function:: PyObject* PyUnicode_DecodeLocaleAndSize(const char *str, \ - Py_ssize_t len, \ + Py_ssize_t length, \ const char *errors) Decode a string from UTF-8 on Android and VxWorks, or from the current @@ -788,7 +788,7 @@ conversion function: Accepts a :term:`path-like object`. -.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *str, Py_ssize_t size) Decode a string from the :term:`filesystem encoding and error handler`. @@ -804,7 +804,7 @@ conversion function: handler>` is now used. -.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s) +.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *str) Decode a null-terminated string from the :term:`filesystem encoding and error handler`. @@ -841,17 +841,17 @@ wchar_t Support :c:type:`wchar_t` support for platforms which support it: -.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *w, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *wstr, Py_ssize_t size) - Create a Unicode object from the :c:type:`wchar_t` buffer *w* of the given *size*. + Create a Unicode object from the :c:type:`wchar_t` buffer *wstr* of the given *size*. Passing ``-1`` as the *size* indicates that the function must itself compute the length, - using wcslen. + using :c:func:`!wcslen`. Return ``NULL`` on failure. -.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) +.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *wstr, Py_ssize_t size) - Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most + Copy the Unicode object contents into the :c:type:`wchar_t` buffer *wstr*. At most *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing null termination character). Return the number of :c:type:`wchar_t` characters copied or ``-1`` in case of an error. Note that the resulting :c:expr:`wchar_t*` @@ -915,10 +915,10 @@ Generic Codecs These are the generic codec APIs: -.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_Decode(const char *str, Py_ssize_t size, \ const char *encoding, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s*. + Create a Unicode object by decoding *size* bytes of the encoded string *str*. *encoding* and *errors* have the same meaning as the parameters of the same name in the :func:`str` built-in function. The codec to be used is looked up using the Python codec registry. Return ``NULL`` if an exception was raised by @@ -941,13 +941,13 @@ UTF-8 Codecs These are the UTF-8 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-8 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF8`. If @@ -1004,7 +1004,7 @@ UTF-32 Codecs These are the UTF-32 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder) Decode *size* bytes from a UTF-32 encoded buffer string and return the @@ -1031,7 +1031,7 @@ These are the UTF-32 codec APIs: Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF32`. If @@ -1054,7 +1054,7 @@ UTF-16 Codecs These are the UTF-16 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder) Decode *size* bytes from a UTF-16 encoded buffer string and return the @@ -1082,7 +1082,7 @@ These are the UTF-16 codec APIs: Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF16`. If @@ -1105,13 +1105,13 @@ UTF-7 Codecs These are the UTF-7 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-7 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF7`. If @@ -1126,11 +1126,11 @@ Unicode-Escape Codecs These are the "Unicode Escape" codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, \ +.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *str, \ Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded - string *s*. Return ``NULL`` if an exception was raised by the codec. + string *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode) @@ -1146,11 +1146,11 @@ Raw-Unicode-Escape Codecs These are the "Raw Unicode Escape" codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, \ +.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *str, \ Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape - encoded string *s*. Return ``NULL`` if an exception was raised by the codec. + encoded string *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) @@ -1167,10 +1167,10 @@ These are the Latin-1 codec APIs: Latin-1 corresponds to the first 256 Unicode ordinals and only these are accepted by the codecs during encoding. -.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Latin-1 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode) @@ -1187,10 +1187,10 @@ These are the ASCII codec APIs. Only 7-bit ASCII data is accepted. All other codes generate errors. -.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the ASCII encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode) @@ -1211,10 +1211,10 @@ decode characters. The mapping objects provided must support the These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *str, Py_ssize_t length, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* + Create a Unicode object by decoding *size* bytes of the encoded string *str* using the given *mapping* object. Return ``NULL`` if an exception was raised by the codec. @@ -1241,7 +1241,7 @@ These are the mapping codec APIs: The following codec API is special in that maps Unicode to Unicode. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors) +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, PyObject *table, const char *errors) Translate a string by applying a character mapping table to it and return the resulting Unicode object. Return ``NULL`` if an exception was raised by the @@ -1266,13 +1266,13 @@ use the Win32 MBCS converters to implement the conversions. Note that MBCS (or DBCS) is a class of encodings, not just one. The target encoding is defined by the user settings on the machine running the codec. -.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *str, Py_ssize_t size, const char *errors) - Create a Unicode object by decoding *size* bytes of the MBCS encoded string *s*. + Create a Unicode object by decoding *size* bytes of the MBCS encoded string *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeMBCS`. If @@ -1318,7 +1318,7 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Concat two strings giving a new Unicode string. -.. c:function:: PyObject* PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) +.. c:function:: PyObject* PyUnicode_Split(PyObject *unicode, PyObject *sep, Py_ssize_t maxsplit) Split a string giving a list of Unicode strings. If *sep* is ``NULL``, splitting will be done at all whitespace substrings. Otherwise, splits occur at the given @@ -1326,10 +1326,10 @@ They all return ``NULL`` or ``-1`` if an exception occurs. set. Separators are not included in the resulting list. -.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *s, int keepend) +.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *unicode, int keepends) Split a Unicode string at line breaks, returning a list of Unicode strings. - CRLF is considered to be one line break. If *keepend* is ``0``, the line break + CRLF is considered to be one line break. If *keepends* is ``0``, the Line break characters are not included in the resulting strings. @@ -1339,28 +1339,28 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Unicode string. -.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return ``1`` if *substr* matches ``str[start:end]`` at the given tail end + Return ``1`` if *substr* matches ``unicode[start:end]`` at the given tail end (*direction* == ``-1`` means to do a prefix match, *direction* == ``1`` a suffix match), ``0`` otherwise. Return ``-1`` if an error occurred. -.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return the first position of *substr* in ``str[start:end]`` using the given + Return the first position of *substr* in ``unicode[start:end]`` using the given *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a backward search). The return value is the index of the first match; a value of ``-1`` indicates that no match was found, and ``-2`` indicates that an error occurred and an exception has been set. -.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, \ +.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *unicode, Py_UCS4 ch, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return the first position of the character *ch* in ``str[start:end]`` using + Return the first position of the character *ch* in ``unicode[start:end]`` using the given *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a backward search). The return value is the index of the first match; a value of ``-1`` indicates that no match was found, and ``-2`` @@ -1369,20 +1369,20 @@ They all return ``NULL`` or ``-1`` if an exception occurs. .. versionadded:: 3.3 .. versionchanged:: 3.7 - *start* and *end* are now adjusted to behave like ``str[start:end]``. + *start* and *end* are now adjusted to behave like ``unicode[start:end]``. -.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end) Return the number of non-overlapping occurrences of *substr* in - ``str[start:end]``. Return ``-1`` if an error occurred. + ``unicode[start:end]``. Return ``-1`` if an error occurred. -.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, \ +.. c:function:: PyObject* PyUnicode_Replace(PyObject *unicode, PyObject *substr, \ PyObject *replstr, Py_ssize_t maxcount) - Replace at most *maxcount* occurrences of *substr* in *str* with *replstr* and + Replace at most *maxcount* occurrences of *substr* in *unicode* with *replstr* and return the resulting Unicode object. *maxcount* == ``-1`` means replace all occurrences. @@ -1418,9 +1418,9 @@ They all return ``NULL`` or ``-1`` if an exception occurs. .. versionadded:: 3.13 -.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) +.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *unicode, const char *string) - Compare a Unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less + Compare a Unicode object, *unicode*, with *string* and return ``-1``, ``0``, ``1`` for less than, equal, and greater than, respectively. It is best to pass only ASCII-encoded strings, but the function interprets the input string as ISO-8859-1 if it contains non-ASCII characters. @@ -1428,7 +1428,7 @@ They all return ``NULL`` or ``-1`` if an exception occurs. This function does not raise exceptions. -.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) +.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) Rich compare two Unicode strings and return one of the following: @@ -1446,29 +1446,29 @@ They all return ``NULL`` or ``-1`` if an exception occurs. ``format % args``. -.. c:function:: int PyUnicode_Contains(PyObject *container, PyObject *element) +.. c:function:: int PyUnicode_Contains(PyObject *unicode, PyObject *substr) - Check whether *element* is contained in *container* and return true or false + Check whether *substr* is contained in *unicode* and return true or false accordingly. - *element* has to coerce to a one element Unicode string. ``-1`` is returned + *substr* has to coerce to a one element Unicode string. ``-1`` is returned if there was an error. -.. c:function:: void PyUnicode_InternInPlace(PyObject **string) +.. c:function:: void PyUnicode_InternInPlace(PyObject **p_unicode) - Intern the argument *\*string* in place. The argument must be the address of a + Intern the argument :c:expr:`*p_unicode` in place. The argument must be the address of a pointer variable pointing to a Python Unicode string object. If there is an - existing interned string that is the same as *\*string*, it sets *\*string* to + existing interned string that is the same as :c:expr:`*p_unicode`, it sets :c:expr:`*p_unicode` to it (releasing the reference to the old string object and creating a new :term:`strong reference` to the interned string object), otherwise it leaves - *\*string* alone and interns it (creating a new :term:`strong reference`). + :c:expr:`*p_unicode` alone and interns it (creating a new :term:`strong reference`). (Clarification: even though there is a lot of talk about references, think of this function as reference-neutral; you own the object after the call if and only if you owned it before the call.) -.. c:function:: PyObject* PyUnicode_InternFromString(const char *v) +.. c:function:: PyObject* PyUnicode_InternFromString(const char *str) A combination of :c:func:`PyUnicode_FromString` and :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string diff --git a/Doc/conf.py b/Doc/conf.py index f1b411126c4e87..dc09b0b51ca84c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -24,7 +24,13 @@ 'sphinx.ext.doctest', ] -# Skip if downstream redistributors haven't installed it +# Skip if downstream redistributors haven't installed them +try: + import notfound.extension +except ImportError: + pass +else: + extensions.append('notfound.extension') try: import sphinxext.opengraph except ImportError: @@ -157,6 +163,13 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), + # Deprecated function that was never documented: + ('py:func', 'getargspec'), + ('py:func', 'inspect.getargspec'), + # Undocumented modules that users shouldn't have to worry about + # (implementation details of `os.path`): + ('py:mod', 'ntpath'), + ('py:mod', 'posixpath'), ] # Temporary undocumented names. @@ -232,6 +245,12 @@ # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. ('py:meth', '_SubParsersAction.add_parser'), + # Attributes/methods/etc. that definitely should be documented better, + # but are deferred for now: + ('py:attr', '__annotations__'), + ('py:meth', '__missing__'), + ('py:attr', '__wrapped__'), + ('py:meth', 'index'), # list.index, tuple.index, etc. ] # gh-106948: Copy standard C types declared in the "c:type" domain to the diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 1ee7f28b2ba220..745fc10a22d161 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -735,7 +735,7 @@ Keyword Parameters for Extension Functions The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: int PyArg_ParseTupleAndKeywords(PyObject *arg, PyObject *kwdict, - const char *format, char *kwlist[], ...); + const char *format, char * const *kwlist, ...); The *arg* and *format* parameters are identical to those of the :c:func:`PyArg_ParseTuple` function. The *kwdict* parameter is the dictionary of diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 6b517b95f97013..601443d5aade94 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -151,9 +151,9 @@ Glossary A :term:`file object` able to read and write :term:`bytes-like objects `. Examples of binary files are files opened in binary mode (``'rb'``, - ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, - :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and - :class:`gzip.GzipFile`. + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer `, + :data:`sys.stdout.buffer `, and instances of + :class:`io.BytesIO` and :class:`gzip.GzipFile`. See also :term:`text file` for a file object able to read and write :class:`str` objects. @@ -304,8 +304,9 @@ Glossary :ref:`class definitions ` for more about decorators. descriptor - Any object which defines the methods :meth:`__get__`, :meth:`__set__`, or - :meth:`__delete__`. When a class attribute is a descriptor, its special + Any object which defines the methods :meth:`~object.__get__`, + :meth:`~object.__set__`, or :meth:`~object.__delete__`. + When a class attribute is a descriptor, its special binding behavior is triggered upon attribute lookup. Normally, using *a.b* to get, set or delete an attribute looks up the object named *b* in the class dictionary for *a*, but if *b* is a descriptor, the respective @@ -319,7 +320,8 @@ Glossary dictionary An associative array, where arbitrary keys are mapped to values. The - keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods. + keys can be any object with :meth:`~object.__hash__` and + :meth:`~object.__eq__` methods. Called a hash in Perl. dictionary comprehension @@ -383,7 +385,7 @@ Glossary file object An object exposing a file-oriented API (with methods such as - :meth:`read()` or :meth:`write()`) to an underlying resource. Depending + :meth:`!read` or :meth:`!write`) to an underlying resource. Depending on the way it was created, a file object can mediate access to a real on-disk file or to another type of storage or communication device (for example standard input/output, in-memory buffers, sockets, pipes, @@ -502,7 +504,7 @@ Glossary .. index:: single: generator expression generator expression - An expression that returns an iterator. It looks like a normal expression + An :term:`expression` that returns an :term:`iterator`. It looks like a normal expression followed by a :keyword:`!for` clause defining a loop variable, range, and an optional :keyword:`!if` clause. The combined expression generates values for an enclosing function:: @@ -559,8 +561,9 @@ Glossary hashable An object is *hashable* if it has a hash value which never changes during - its lifetime (it needs a :meth:`__hash__` method), and can be compared to - other objects (it needs an :meth:`__eq__` method). Hashable objects which + its lifetime (it needs a :meth:`~object.__hash__` method), and can be + compared to other objects (it needs an :meth:`~object.__eq__` method). + Hashable objects which compare equal must have the same hash value. Hashability makes an object usable as a dictionary key and a set member, @@ -646,7 +649,8 @@ Glossary iterables include all sequence types (such as :class:`list`, :class:`str`, and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects `, and objects of any classes you define - with an :meth:`__iter__` method or with a :meth:`~object.__getitem__` method + with an :meth:`~iterator.__iter__` method or with a + :meth:`~object.__getitem__` method that implements :term:`sequence` semantics. Iterables can be @@ -655,7 +659,7 @@ Glossary as an argument to the built-in function :func:`iter`, it returns an iterator for the object. This iterator is good for one pass over the set of values. When using iterables, it is usually not necessary to call - :func:`iter` or deal with iterator objects yourself. The ``for`` + :func:`iter` or deal with iterator objects yourself. The :keyword:`for` statement does that automatically for you, creating a temporary unnamed variable to hold the iterator for the duration of the loop. See also :term:`iterator`, :term:`sequence`, and :term:`generator`. @@ -666,8 +670,8 @@ Glossary :func:`next`) return successive items in the stream. When no more data are available a :exc:`StopIteration` exception is raised instead. At this point, the iterator object is exhausted and any further calls to its - :meth:`__next__` method just raise :exc:`StopIteration` again. Iterators - are required to have an :meth:`__iter__` method that returns the iterator + :meth:`!__next__` method just raise :exc:`StopIteration` again. Iterators + are required to have an :meth:`~iterator.__iter__` method that returns the iterator object itself so every iterator is also iterable and may be used in most places where other iterables are accepted. One notable exception is code which attempts multiple iteration passes. A container object (such as a @@ -681,7 +685,7 @@ Glossary .. impl-detail:: CPython does not consistently apply the requirement that an iterator - define :meth:`__iter__`. + define :meth:`~iterator.__iter__`. key function A key function or collation function is a callable that returns a value @@ -875,7 +879,8 @@ Glossary Old name for the flavor of classes now used for all class objects. In earlier Python versions, only new-style classes could use Python's newer, versatile features like :attr:`~object.__slots__`, descriptors, - properties, :meth:`__getattribute__`, class methods, and static methods. + properties, :meth:`~object.__getattribute__`, class methods, and static + methods. object Any data with state (attributes or value) and defined behavior @@ -955,7 +960,7 @@ Glossary finders implement. path entry hook - A callable on the :data:`sys.path_hook` list which returns a :term:`path + A callable on the :data:`sys.path_hooks` list which returns a :term:`path entry finder` if it knows how to find modules on a specific :term:`path entry`. @@ -1089,18 +1094,18 @@ Glossary sequence An :term:`iterable` which supports efficient element access using integer indices via the :meth:`~object.__getitem__` special method and defines a - :meth:`__len__` method that returns the length of the sequence. + :meth:`~object.__len__` method that returns the length of the sequence. Some built-in sequence types are :class:`list`, :class:`str`, :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also - supports :meth:`~object.__getitem__` and :meth:`__len__`, but is considered a + supports :meth:`~object.__getitem__` and :meth:`!__len__`, but is considered a mapping rather than a sequence because the lookups use arbitrary :term:`immutable` keys rather than integers. The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just - :meth:`~object.__getitem__` and :meth:`__len__`, adding :meth:`count`, - :meth:`index`, :meth:`__contains__`, and - :meth:`__reversed__`. Types that implement this expanded + :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding + :meth:`count`, :meth:`index`, :meth:`~object.__contains__`, and + :meth:`~object.__reversed__`. Types that implement this expanded interface can be registered explicitly using :func:`~abc.ABCMeta.register`. diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 1134686c947d66..be8c7e6c827f57 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -153,7 +153,8 @@ on an arbitrary object ``o``: unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as appropriate, until you have found the root unwrapped function. * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. + :attr:`o.__globals__ ` as the globals when calling + :func:`eval`. However, not all string values used as annotations can be successfully turned into Python values by :func:`eval`. diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index f732aaea729d40..87274a5133d1cf 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1342,7 +1342,8 @@ Using the non-data descriptor protocol, a pure Python version of The :func:`functools.update_wrapper` call adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look like the wrapped -function: ``__name__``, ``__qualname__``, ``__doc__``, and ``__annotations__``. +function: :attr:`~function.__name__`, :attr:`~function.__qualname__`, +:attr:`~function.__doc__`, and :attr:`~function.__annotations__`. .. testcode:: :hide: @@ -1522,8 +1523,9 @@ Using the non-data descriptor protocol, a pure Python version of The :func:`functools.update_wrapper` call in ``ClassMethod`` adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look -like the wrapped function: ``__name__``, ``__qualname__``, ``__doc__``, -and ``__annotations__``. +like the wrapped function: :attr:`~function.__name__`, +:attr:`~function.__qualname__`, :attr:`~function.__doc__`, +and :attr:`~function.__annotations__`. Member objects and __slots__ diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index a136c76303c8ef..1e9ac9b6761b64 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -868,7 +868,7 @@ Others While :class:`IntEnum` is part of the :mod:`enum` module, it would be very simple to implement independently:: - class IntEnum(int, Enum): + class IntEnum(int, ReprEnum): # or Enum instead of ReprEnum pass This demonstrates how similar derived enumerations can be defined; for example @@ -876,8 +876,8 @@ a :class:`FloatEnum` that mixes in :class:`float` instead of :class:`int`. Some rules: -1. When subclassing :class:`Enum`, mix-in types must appear before - :class:`Enum` itself in the sequence of bases, as in the :class:`IntEnum` +1. When subclassing :class:`Enum`, mix-in types must appear before the + :class:`Enum` class itself in the sequence of bases, as in the :class:`IntEnum` example above. 2. Mix-in types must be subclassable. For example, :class:`bool` and :class:`range` are not subclassable and will throw an error during Enum @@ -961,30 +961,34 @@ all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- ``_name_`` -- name of the member -- ``_value_`` -- value of the member; can be set / modified in ``__new__`` +- :attr:`~Enum._name_` -- name of the member +- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` +- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; + may be overridden +- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a + :class:`str`, that will not be transformed into members, and will be removed + from the final class +- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for + an enum member; may be overridden +- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an + existing member. See `MultiValueEnum`_ for an example. -- ``_missing_`` -- a lookup function used when a value is not found; may be - overridden -- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, - that will not be transformed into members, and will be removed from the final - class -- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent - (class attribute, removed during class creation) -- ``_generate_next_value_`` -- used by the `Functional API`_ and by - :class:`auto` to get an appropriate value for an enum member; may be - overridden + .. note:: -.. note:: + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. - For standard :class:`Enum` classes the next value chosen is the last value seen - incremented by one. + For :class:`Flag` classes the next value chosen will be the next highest + power-of-two. - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two, regardless of the last value seen. + .. versionchanged:: 3.13 + Prior versions would use the last seen value instead of the highest value. .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can be provided. It will be checked against the actual order of the enumeration @@ -1439,7 +1443,6 @@ alias:: Traceback (most recent call last): ... ValueError: aliases not allowed in DuplicateFreeEnum: 'GRENE' --> 'GREEN' - Error calling __set_name__ on '_proto_member' instance 'GRENE' in 'Color' .. note:: @@ -1448,6 +1451,29 @@ alias:: disallowing aliases, the :func:`unique` decorator can be used instead. +MultiValueEnum +^^^^^^^^^^^^^^^^^ + +Supports having more than one value per member:: + + >>> class MultiValueEnum(Enum): + ... def __new__(cls, value, *values): + ... self = object.__new__(cls) + ... self._value_ = value + ... for v in values: + ... self._add_value_alias_(v) + ... return self + ... + >>> class DType(MultiValueEnum): + ... float32 = 'f', 8 + ... double64 = 'd', 9 + ... + >>> DType('f') + + >>> DType(9) + + + Planet ^^^^^^ diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 835c0afad7c6c8..e35855deedbe5f 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -337,7 +337,7 @@ That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - Define a traverse function using ``Py_tp_traverse``, which - visits the type (e.g. using :c:expr:`Py_VISIT(Py_TYPE(self))`). + visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the the documentation of :c:macro:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse` @@ -482,7 +482,7 @@ The largest roadblock is getting *the class a method was defined in*, or that method's "defining class" for short. The defining class can have a reference to the module it is part of. -Do not confuse the defining class with :c:expr:`Py_TYPE(self)`. If the method +Do not confuse the defining class with ``Py_TYPE(self)``. If the method is called on a *subclass* of your type, ``Py_TYPE(self)`` will refer to that subclass, which may be defined in different module than yours. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 588f5a0a53ded0..ea494f2fdbbce4 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -332,10 +332,10 @@ Suppose you configure logging with the following JSON: } } -This configuration does *almost* what we want, except that ``sys.stdout`` would -show messages of severity ``ERROR`` and above as well as ``INFO`` and -``WARNING`` messages. To prevent this, we can set up a filter which excludes -those messages and add it to the relevant handler. This can be configured by +This configuration does *almost* what we want, except that ``sys.stdout`` would show messages +of severity ``ERROR`` and only events of this severity and higher will be tracked +as well as ``INFO`` and ``WARNING`` messages. To prevent this, we can set up a filter which +excludes those messages and add it to the relevant handler. This can be configured by adding a ``filters`` section parallel to ``formatters`` and ``handlers``: .. code-block:: json diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 7330cf675baa36..f164b461c93b9c 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -89,9 +89,8 @@ described below (in increasing order of severity): | | itself may be unable to continue running. | +--------------+---------------------------------------------+ -The default level is ``WARNING``, which means that only events of this level -and above will be tracked, unless the logging package is configured to do -otherwise. +The default level is ``WARNING``, which means that only events of this severity and higher +will be tracked, unless the logging package is configured to do otherwise. Events that are tracked can be handled in different ways. The simplest way of handling tracked events is to print them to the console. Another common way diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index fb4f9da169c5ab..c073ea955abaa4 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -21,7 +21,7 @@ The :mod:`collections` module has some concrete classes that derive from ABCs; these can, of course, be further derived. In addition, the :mod:`collections.abc` submodule has some ABCs that can be used to test whether a class or instance provides a particular interface, for example, if it is -:term:`hashable` or if it is a mapping. +:term:`hashable` or if it is a :term:`mapping`. This module provides the metaclass :class:`ABCMeta` for defining ABCs and @@ -30,7 +30,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: .. class:: ABC A helper class that has :class:`ABCMeta` as its metaclass. With this class, - an abstract base class can be created by simply deriving from :class:`ABC` + an abstract base class can be created by simply deriving from :class:`!ABC` avoiding sometimes confusing metaclass usage, for example:: from abc import ABC @@ -38,11 +38,11 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: class MyABC(ABC): pass - Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore - inheriting from :class:`ABC` requires the usual precautions regarding + Note that the type of :class:`!ABC` is still :class:`ABCMeta`, therefore + inheriting from :class:`!ABC` requires the usual precautions regarding metaclass usage, as multiple inheritance may lead to metaclass conflicts. One may also define an abstract base class by passing the metaclass - keyword and using :class:`ABCMeta` directly, for example:: + keyword and using :class:`!ABCMeta` directly, for example:: from abc import ABCMeta @@ -65,7 +65,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: implementations defined by the registering ABC be callable (not even via :func:`super`). [#]_ - Classes created with a metaclass of :class:`ABCMeta` have the following method: + Classes created with a metaclass of :class:`!ABCMeta` have the following method: .. method:: register(subclass) @@ -86,7 +86,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: Returns the registered subclass, to allow usage as a class decorator. .. versionchanged:: 3.4 - To detect calls to :meth:`register`, you can use the + To detect calls to :meth:`!register`, you can use the :func:`get_cache_token` function. You can also override this method in an abstract base class: @@ -96,10 +96,10 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: (Must be defined as a class method.) Check whether *subclass* is considered a subclass of this ABC. This means - that you can customize the behavior of ``issubclass`` further without the + that you can customize the behavior of :func:`issubclass` further without the need to call :meth:`register` on every class you want to consider a subclass of the ABC. (This class method is called from the - :meth:`__subclasscheck__` method of the ABC.) + :meth:`~class.__subclasscheck__` method of the ABC.) This method should return ``True``, ``False`` or ``NotImplemented``. If it returns ``True``, the *subclass* is considered a subclass of this ABC. @@ -142,7 +142,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: The ABC ``MyIterable`` defines the standard iterable method, :meth:`~iterator.__iter__`, as an abstract method. The implementation given - here can still be called from subclasses. The :meth:`get_iterator` method + here can still be called from subclasses. The :meth:`!get_iterator` method is also part of the ``MyIterable`` abstract base class, but it does not have to be overridden in non-abstract derived classes. @@ -153,14 +153,14 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, even though it does not define an :meth:`~iterator.__iter__` method (it uses - the old-style iterable protocol, defined in terms of :meth:`__len__` and + the old-style iterable protocol, defined in terms of :meth:`~object.__len__` and :meth:`~object.__getitem__`). Note that this will not make ``get_iterator`` available as a method of ``Foo``, so it is provided separately. -The :mod:`abc` module also provides the following decorator: +The :mod:`!abc` module also provides the following decorator: .. decorator:: abstractmethod @@ -168,19 +168,19 @@ The :mod:`abc` module also provides the following decorator: Using this decorator requires that the class's metaclass is :class:`ABCMeta` or is derived from it. A class that has a metaclass derived from - :class:`ABCMeta` cannot be instantiated unless all of its abstract methods + :class:`!ABCMeta` cannot be instantiated unless all of its abstract methods and properties are overridden. The abstract methods can be called using any - of the normal 'super' call mechanisms. :func:`abstractmethod` may be used + of the normal 'super' call mechanisms. :func:`!abstractmethod` may be used to declare abstract methods for properties and descriptors. Dynamically adding abstract methods to a class, or attempting to modify the abstraction status of a method or class once it is created, are only supported using the :func:`update_abstractmethods` function. The - :func:`abstractmethod` only affects subclasses derived using regular - inheritance; "virtual subclasses" registered with the ABC's :meth:`register` - method are not affected. + :func:`!abstractmethod` only affects subclasses derived using regular + inheritance; "virtual subclasses" registered with the ABC's + :meth:`~ABCMeta.register` method are not affected. - When :func:`abstractmethod` is applied in combination with other method + When :func:`!abstractmethod` is applied in combination with other method descriptors, it should be applied as the innermost decorator, as shown in the following usage examples:: @@ -216,7 +216,7 @@ The :mod:`abc` module also provides the following decorator: In order to correctly interoperate with the abstract base class machinery, the descriptor must identify itself as abstract using - :attr:`__isabstractmethod__`. In general, this attribute should be ``True`` + :attr:`!__isabstractmethod__`. In general, this attribute should be ``True`` if any of the methods used to compose the descriptor are abstract. For example, Python's built-in :class:`property` does the equivalent of:: @@ -236,7 +236,7 @@ The :mod:`abc` module also provides the following decorator: super-call in a framework that uses cooperative multiple-inheritance. -The :mod:`abc` module also supports the following legacy decorators: +The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractclassmethod @@ -323,7 +323,7 @@ The :mod:`abc` module also supports the following legacy decorators: ... -The :mod:`abc` module also provides the following functions: +The :mod:`!abc` module also provides the following functions: .. function:: get_cache_token() diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 4ebbe0e5471c88..c943c2f498173e 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -45,7 +45,7 @@ Node classes This is the base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced - :ref:`above `. They are defined in the :mod:`_ast` C + :ref:`above `. They are defined in the :mod:`!_ast` C module and re-exported in :mod:`ast`. There is one class defined for each left-hand side symbol in the abstract @@ -128,14 +128,14 @@ Node classes .. deprecated:: 3.8 - Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, - :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, + Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, + :class:`!ast.NameConstant` and :class:`!ast.Ellipsis` are still available, but they will be removed in future Python releases. In the meantime, instantiating them will return an instance of a different class. .. deprecated:: 3.9 - Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still + Old classes :class:`!ast.Index` and :class:`!ast.ExtSlice` are still available, but they will be removed in future Python releases. In the meantime, instantiating them will return an instance of a different class. @@ -1935,8 +1935,7 @@ Function and class definitions .. class:: arg(arg, annotation, type_comment) A single argument in a list. ``arg`` is a raw string of the argument - name, ``annotation`` is its annotation, such as a :class:`Str` or - :class:`Name` node. + name; ``annotation`` is its annotation, such as a :class:`Name` node. .. attribute:: type_comment @@ -2210,7 +2209,7 @@ and classes for traversing abstract syntax trees: Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. .. versionchanged:: 3.13 - The minimum supported version for feature_version is now (3,7) + The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. @@ -2286,8 +2285,8 @@ and classes for traversing abstract syntax trees: .. function:: get_source_segment(source, node, *, padded=False) Get source code segment of the *source* that generated *node*. - If some location information (:attr:`lineno`, :attr:`end_lineno`, - :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. + If some location information (:attr:`~ast.AST.lineno`, :attr:`~ast.AST.end_lineno`, + :attr:`~ast.AST.col_offset`, or :attr:`~ast.AST.end_col_offset`) is missing, return ``None``. If *padded* is ``True``, the first line of a multi-line statement will be padded with spaces to match its original position. @@ -2298,7 +2297,7 @@ and classes for traversing abstract syntax trees: .. function:: fix_missing_locations(node) When you compile a node tree with :func:`compile`, the compiler expects - :attr:`lineno` and :attr:`col_offset` attributes for every node that supports + :attr:`~ast.AST.lineno` and :attr:`~ast.AST.col_offset` attributes for every node that supports them. This is rather tedious to fill in for generated nodes, so this helper adds these attributes recursively where not already set, by setting them to the values of the parent node. It works recursively starting at *node*. @@ -2313,8 +2312,8 @@ and classes for traversing abstract syntax trees: .. function:: copy_location(new_node, old_node) - Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, - and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, + Copy source location (:attr:`~ast.AST.lineno`, :attr:`~ast.AST.col_offset`, :attr:`~ast.AST.end_lineno`, + and :attr:`~ast.AST.end_col_offset`) from *old_node* to *new_node* if possible, and return *new_node*. @@ -2360,14 +2359,18 @@ and classes for traversing abstract syntax trees: visited unless the visitor calls :meth:`generic_visit` or visits them itself. + .. method:: visit_Constant(node) + + Handles all constant nodes. + Don't use the :class:`NodeVisitor` if you want to apply changes to nodes during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. .. deprecated:: 3.8 - Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, - :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated + Methods :meth:`!visit_Num`, :meth:`!visit_Str`, :meth:`!visit_Bytes`, + :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` are deprecated now and will not be called in future Python versions. Add the :meth:`visit_Constant` method to handle all constant nodes. @@ -2396,7 +2399,7 @@ and classes for traversing abstract syntax trees: ) Keep in mind that if the node you're operating on has child nodes you must - either transform the child nodes yourself or call the :meth:`generic_visit` + either transform the child nodes yourself or call the :meth:`~ast.NodeVisitor.generic_visit` method for the node first. For nodes that were part of a collection of statements (that applies to all @@ -2405,7 +2408,7 @@ and classes for traversing abstract syntax trees: If :class:`NodeTransformer` introduces new nodes (that weren't part of original tree) without giving them location information (such as - :attr:`lineno`), :func:`fix_missing_locations` should be called with + :attr:`~ast.AST.lineno`), :func:`fix_missing_locations` should be called with the new sub-tree to recalculate the location information:: tree = ast.parse('foo', mode='eval') @@ -2457,6 +2460,13 @@ effects on the compilation of a program: Generates and returns an abstract syntax tree instead of returning a compiled code object. +.. data:: PyCF_OPTIMIZED_AST + + The returned AST is optimized according to the *optimize* argument + in :func:`compile` or :func:`ast.parse`. + + .. versionadded:: 3.13 + .. data:: PyCF_TYPE_COMMENTS Enables support for :pep:`484` and :pep:`526` style type comments diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index ea1d146f06cf2b..828e506a72c937 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -671,6 +671,7 @@ Creating network servers flags=socket.AI_PASSIVE, \ sock=None, backlog=100, ssl=None, \ reuse_address=None, reuse_port=None, \ + keep_alive=None, \ ssl_handshake_timeout=None, \ ssl_shutdown_timeout=None, \ start_serving=True) @@ -735,6 +736,13 @@ Creating network servers set this flag when being created. This option is not supported on Windows. + * *keep_alive* set to ``True`` keeps connections active by enabling the + periodic transmission of messages. + + .. versionchanged:: 3.13 + + Added the *keep_alive* parameter. + * *ssl_handshake_timeout* is (for a TLS server) the time in seconds to wait for the TLS handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index d201dc963b5995..4ce5c9bcde38ff 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -294,7 +294,7 @@ The :mod:`bdb` module also defines two classes: .. method:: set_quit() Set the :attr:`quitting` attribute to ``True``. This raises :exc:`BdbQuit` in - the next call to one of the :meth:`dispatch_\*` methods. + the next call to one of the :meth:`!dispatch_\*` methods. Derived classes and clients can call the following methods to manipulate diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 8022c596f0af97..c0923093c1cb06 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -19,9 +19,9 @@ linear searches or frequent resorting. The module is called :mod:`bisect` because it uses a basic bisection algorithm to do its work. Unlike other bisection tools that search for a specific value, the functions in this module are designed to locate an -insertion point. Accordingly, the functions never call an :meth:`__eq__` +insertion point. Accordingly, the functions never call an :meth:`~object.__eq__` method to determine whether a value has been found. Instead, the -functions only call the :meth:`__lt__` method and will return an insertion +functions only call the :meth:`~object.__lt__` method and will return an insertion point between values in an array. .. _bisect functions: @@ -73,7 +73,7 @@ The following functions are provided: Insert *x* in *a* in sorted order. This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. - Next, it runs the :meth:`insert` method on *a* to insert *x* at the + Next, it runs the :meth:`!insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is @@ -93,7 +93,7 @@ The following functions are provided: entries of *x*. This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. - Next, it runs the :meth:`insert` method on *a* to insert *x* at the + Next, it runs the :meth:`!insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 157a7537f97dc6..6586f539a8da4f 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -196,6 +196,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is output (defaulting to the system default encoding). + .. method:: formatmonthname(theyear, themonth, withyear=True) + + Return a month name as an HTML table row. If *withyear* is true the year + will be included in the row, otherwise just the month name will be + used. + + :class:`!HTMLCalendar` has the following attributes you can override to customize the CSS classes used by the calendar: @@ -289,7 +296,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. note:: - The constructor, :meth:`formatweekday` and :meth:`formatmonthname` methods + The constructor, :meth:`!formatweekday` and :meth:`!formatmonthname` methods of these two classes temporarily change the ``LC_TIME`` locale to the given *locale*. Because the current locale is a process-wide setting, they are not thread-safe. @@ -358,7 +365,7 @@ For simple text calendars this module provides the following functions. .. function:: month(theyear, themonth, w=0, l=0) - Returns a month's calendar in a multi-line string using the :meth:`formatmonth` + Returns a month's calendar in a multi-line string using the :meth:`~TextCalendar.formatmonth` of the :class:`TextCalendar` class. @@ -370,7 +377,7 @@ For simple text calendars this module provides the following functions. .. function:: calendar(year, w=2, l=1, c=6, m=3) Returns a 3-column calendar for an entire year as a multi-line string using - the :meth:`formatyear` of the :class:`TextCalendar` class. + the :meth:`~TextCalendar.formatyear` of the :class:`TextCalendar` class. .. function:: timegm(tuple) diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index fd5df96dfd0b3d..39ef4b481478d1 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -26,6 +26,13 @@ interface. key; it defaults to :kbd:`Tab`. If *completekey* is not :const:`None` and :mod:`readline` is available, command completion is done automatically. + The default, ``'tab'``, is treated specially, so that it refers to the + :kbd:`Tab` key on every :data:`readline.backend`. + Specifically, if :data:`readline.backend` is ``editline``, + ``Cmd`` will use ``'^I'`` instead of ``'tab'``. + Note that other values are not treated this way, and might only work + with a specific backend. + The optional arguments *stdin* and *stdout* specify the input and output file objects that the Cmd instance or subclass instance will use for input and output. If not specified, they will default to :data:`sys.stdin` and @@ -35,6 +42,9 @@ interface. :attr:`use_rawinput` attribute to ``False``, otherwise *stdin* will be ignored. + .. versionchanged:: 3.13 + ``completekey='tab'`` is replaced by ``'^I'`` for ``editline``. + .. _cmd-objects: @@ -66,29 +76,32 @@ A :class:`Cmd` instance has the following methods: single: ! (exclamation); in a command interpreter An interpreter instance will recognize a command name ``foo`` if and only if it - has a method :meth:`do_foo`. As a special case, a line beginning with the + has a method :meth:`!do_foo`. As a special case, a line beginning with the character ``'?'`` is dispatched to the method :meth:`do_help`. As another special case, a line beginning with the character ``'!'`` is dispatched to the - method :meth:`do_shell` (if such a method is defined). + method :meth:`!do_shell` (if such a method is defined). This method will return when the :meth:`postcmd` method returns a true value. The *stop* argument to :meth:`postcmd` is the return value from the command's - corresponding :meth:`do_\*` method. + corresponding :meth:`!do_\*` method. If completion is enabled, completing commands will be done automatically, and - completing of commands args is done by calling :meth:`complete_foo` with + completing of commands args is done by calling :meth:`!complete_foo` with arguments *text*, *line*, *begidx*, and *endidx*. *text* is the string prefix we are attempting to match: all returned matches must begin with it. *line* is the current input line with leading whitespace removed, *begidx* and *endidx* are the beginning and ending indexes of the prefix text, which could be used to provide different completion depending upon which position the argument is in. - All subclasses of :class:`Cmd` inherit a predefined :meth:`do_help`. This + +.. method:: Cmd.do_help(arg) + + All subclasses of :class:`Cmd` inherit a predefined :meth:`!do_help`. This method, called with an argument ``'bar'``, invokes the corresponding method - :meth:`help_bar`, and if that is not present, prints the docstring of - :meth:`do_bar`, if available. With no argument, :meth:`do_help` lists all + :meth:`!help_bar`, and if that is not present, prints the docstring of + :meth:`!do_bar`, if available. With no argument, :meth:`!do_help` lists all available help topics (that is, all commands with corresponding - :meth:`help_\*` methods or commands that have docstrings), and also lists any + :meth:`!help_\*` methods or commands that have docstrings), and also lists any undocumented commands. @@ -98,7 +111,7 @@ A :class:`Cmd` instance has the following methods: This may be overridden, but should not normally need to be; see the :meth:`precmd` and :meth:`postcmd` methods for useful execution hooks. The return value is a flag indicating whether interpretation of commands by the - interpreter should stop. If there is a :meth:`do_\*` method for the command + interpreter should stop. If there is a :meth:`!do_\*` method for the command *str*, the return value of that method is returned, otherwise the return value from the :meth:`default` method is returned. @@ -118,7 +131,7 @@ A :class:`Cmd` instance has the following methods: .. method:: Cmd.completedefault(text, line, begidx, endidx) Method called to complete an input line when no command-specific - :meth:`complete_\*` method is available. By default, it returns an empty list. + :meth:`!complete_\*` method is available. By default, it returns an empty list. .. method:: Cmd.columnize(list, displaywidth=80) @@ -199,14 +212,14 @@ Instances of :class:`Cmd` subclasses have some public instance variables: .. attribute:: Cmd.misc_header The header to issue if the help output has a section for miscellaneous help - topics (that is, there are :meth:`help_\*` methods without corresponding - :meth:`do_\*` methods). + topics (that is, there are :meth:`!help_\*` methods without corresponding + :meth:`!do_\*` methods). .. attribute:: Cmd.undoc_header The header to issue if the help output has a section for undocumented commands - (that is, there are :meth:`do_\*` methods without corresponding :meth:`help_\*` + (that is, there are :meth:`!do_\*` methods without corresponding :meth:`!help_\*` methods). @@ -219,8 +232,8 @@ Instances of :class:`Cmd` subclasses have some public instance variables: .. attribute:: Cmd.use_rawinput A flag, defaulting to true. If true, :meth:`cmdloop` uses :func:`input` to - display a prompt and read the next command; if false, :meth:`sys.stdout.write` - and :meth:`sys.stdin.readline` are used. (This means that by importing + display a prompt and read the next command; if false, :data:`sys.stdout.write() ` + and :data:`sys.stdin.readline() ` are used. (This means that by importing :mod:`readline`, on systems that support it, the interpreter will automatically support :program:`Emacs`\ -like line editing and command-history keystrokes.) @@ -239,14 +252,14 @@ This section presents a simple example of how to build a shell around a few of the commands in the :mod:`turtle` module. Basic turtle commands such as :meth:`~turtle.forward` are added to a -:class:`Cmd` subclass with method named :meth:`do_forward`. The argument is +:class:`Cmd` subclass with method named :meth:`!do_forward`. The argument is converted to a number and dispatched to the turtle module. The docstring is used in the help utility provided by the shell. The example also includes a basic record and playback facility implemented with the :meth:`~Cmd.precmd` method which is responsible for converting the input to -lowercase and writing the commands to a file. The :meth:`do_playback` method -reads the file and adds the recorded commands to the :attr:`cmdqueue` for +lowercase and writing the commands to a file. The :meth:`!do_playback` method +reads the file and adds the recorded commands to the :attr:`~Cmd.cmdqueue` for immediate playback:: import cmd, sys diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index edc078953290d7..582bb18f752bd5 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -22,7 +22,7 @@ This module provides :term:`abstract base classes ` that can be used to test whether a class provides a particular interface; for -example, whether it is :term:`hashable` or whether it is a mapping. +example, whether it is :term:`hashable` or whether it is a :term:`mapping`. An :func:`issubclass` or :func:`isinstance` test for an interface works in one of three ways. @@ -73,7 +73,7 @@ of the API: >>> isinstance(D(), Sequence) True -In this example, class :class:`D` does not need to define +In this example, class :class:`!D` does not need to define ``__contains__``, ``__iter__``, and ``__reversed__`` because the :ref:`in-operator `, the :term:`iteration ` logic, and the :func:`reversed` function automatically fall back to @@ -87,7 +87,7 @@ the required methods (unless those methods have been set to class E: def __iter__(self): ... - def __next__(next): ... + def __next__(self): ... .. doctest:: @@ -183,14 +183,14 @@ ABC Inherits from Abstract Methods Mi .. rubric:: Footnotes -.. [1] These ABCs override :meth:`object.__subclasshook__` to support +.. [1] These ABCs override :meth:`~abc.ABCMeta.__subclasshook__` to support testing an interface by verifying the required methods are present and have not been set to :const:`None`. This only works for simple interfaces. More complex interfaces require registration or direct subclassing. .. [2] Checking ``isinstance(obj, Iterable)`` detects classes that are - registered as :class:`Iterable` or that have an :meth:`__iter__` + registered as :class:`Iterable` or that have an :meth:`~container.__iter__` method, but it does not detect classes that iterate with the :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -202,26 +202,27 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Container - ABC for classes that provide the :meth:`__contains__` method. + ABC for classes that provide the :meth:`~object.__contains__` method. .. class:: Hashable - ABC for classes that provide the :meth:`__hash__` method. + ABC for classes that provide the :meth:`~object.__hash__` method. .. class:: Sized - ABC for classes that provide the :meth:`__len__` method. + ABC for classes that provide the :meth:`~object.__len__` method. .. class:: Callable - ABC for classes that provide the :meth:`__call__` method. + ABC for classes that provide the :meth:`~object.__call__` method. .. class:: Iterable - ABC for classes that provide the :meth:`__iter__` method. + ABC for classes that provide the :meth:`~container.__iter__` method. Checking ``isinstance(obj, Iterable)`` detects classes that are registered - as :class:`Iterable` or that have an :meth:`__iter__` method, but it does + as :class:`Iterable` or that have an :meth:`~container.__iter__` method, + but it does not detect classes that iterate with the :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -240,17 +241,17 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Reversible - ABC for iterable classes that also provide the :meth:`__reversed__` + ABC for iterable classes that also provide the :meth:`~object.__reversed__` method. .. versionadded:: 3.6 .. class:: Generator - ABC for generator classes that implement the protocol defined in - :pep:`342` that extends iterators with the :meth:`~generator.send`, + ABC for :term:`generator` classes that implement the protocol defined in + :pep:`342` that extends :term:`iterators ` with the + :meth:`~generator.send`, :meth:`~generator.throw` and :meth:`~generator.close` methods. - See also the definition of :term:`generator`. .. versionadded:: 3.5 @@ -261,7 +262,7 @@ Collections Abstract Base Classes -- Detailed Descriptions ABCs for read-only and mutable :term:`sequences `. Implementation note: Some of the mixin methods, such as - :meth:`__iter__`, :meth:`__reversed__` and :meth:`index`, make + :meth:`~container.__iter__`, :meth:`~object.__reversed__` and :meth:`index`, make repeated calls to the underlying :meth:`~object.__getitem__` method. Consequently, if :meth:`~object.__getitem__` is implemented with constant access speed, the mixin methods will have linear performance; @@ -282,7 +283,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Set MutableSet - ABCs for read-only and mutable sets. + ABCs for read-only and mutable :ref:`sets `. .. class:: Mapping MutableMapping @@ -299,16 +300,16 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Awaitable ABC for :term:`awaitable` objects, which can be used in :keyword:`await` - expressions. Custom implementations must provide the :meth:`__await__` - method. + expressions. Custom implementations must provide the + :meth:`~object.__await__` method. :term:`Coroutine ` objects and instances of the :class:`~collections.abc.Coroutine` ABC are all instances of this ABC. .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. + In CPython, generator-based coroutines (:term:`generators ` + decorated with :func:`@types.coroutine `) are + *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -316,17 +317,17 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Coroutine - ABC for coroutine compatible classes. These implement the + ABC for :term:`coroutine` compatible classes. These implement the following methods, defined in :ref:`coroutine-objects`: :meth:`~coroutine.send`, :meth:`~coroutine.throw`, and :meth:`~coroutine.close`. Custom implementations must also implement - :meth:`__await__`. All :class:`Coroutine` instances are also instances of - :class:`Awaitable`. See also the definition of :term:`coroutine`. + :meth:`~object.__await__`. All :class:`Coroutine` instances are also + instances of :class:`Awaitable`. .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. + In CPython, generator-based coroutines (:term:`generators ` + decorated with :func:`@types.coroutine `) are + *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -334,7 +335,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: AsyncIterable - ABC for classes that provide ``__aiter__`` method. See also the + ABC for classes that provide an ``__aiter__`` method. See also the definition of :term:`asynchronous iterable`. .. versionadded:: 3.5 @@ -348,7 +349,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: AsyncGenerator - ABC for asynchronous generator classes that implement the protocol + ABC for :term:`asynchronous generator` classes that implement the protocol defined in :pep:`525` and :pep:`492`. .. versionadded:: 3.6 @@ -373,9 +374,9 @@ particular functionality, for example:: Several of the ABCs are also useful as mixins that make it easier to develop classes supporting container APIs. For example, to write a class supporting the full :class:`Set` API, it is only necessary to supply the three underlying -abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`. -The ABC supplies the remaining methods such as :meth:`__and__` and -:meth:`isdisjoint`:: +abstract methods: :meth:`~object.__contains__`, :meth:`~container.__iter__`, and +:meth:`~object.__len__`. The ABC supplies the remaining methods such as +:meth:`!__and__` and :meth:`~frozenset.isdisjoint`:: class ListBasedSet(collections.abc.Set): ''' Alternate set implementation favoring space over speed @@ -403,23 +404,24 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: (1) Since some set operations create new sets, the default mixin methods need - a way to create new instances from an iterable. The class constructor is + a way to create new instances from an :term:`iterable`. The class constructor is assumed to have a signature in the form ``ClassName(iterable)``. - That assumption is factored-out to an internal classmethod called - :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set. + That assumption is factored-out to an internal :class:`classmethod` called + :meth:`!_from_iterable` which calls ``cls(iterable)`` to produce a new set. If the :class:`Set` mixin is being used in a class with a different - constructor signature, you will need to override :meth:`_from_iterable` + constructor signature, you will need to override :meth:`!_from_iterable` with a classmethod or regular method that can construct new instances from an iterable argument. (2) To override the comparisons (presumably for speed, as the - semantics are fixed), redefine :meth:`__le__` and :meth:`__ge__`, + semantics are fixed), redefine :meth:`~object.__le__` and + :meth:`~object.__ge__`, then the other operations will automatically follow suit. (3) - The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value - for the set; however, :meth:`__hash__` is not defined because not all sets + The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value + for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, inherit from both :meth:`Set` and :meth:`Hashable`, then define ``__hash__ = Set._hash``. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 17dd6da7479e50..233b2c6a771f4a 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -981,6 +981,10 @@ field names, the method and attribute names start with an underscore. Named tuples are also supported by generic function :func:`copy.replace`. + .. versionchanged:: 3.13 + Raise :exc:`TypeError` instead of :exc:`ValueError` for invalid + keyword arguments. + .. attribute:: somenamedtuple._fields Tuple of strings listing the field names. Useful for introspection diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index bb282428c5fffc..0031737853e7b4 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -208,7 +208,7 @@ converters and customize the provided ones. [1]_ Fallback Values --------------- -As with a dictionary, you can use a section's :meth:`get` method to +As with a dictionary, you can use a section's :meth:`~ConfigParser.get` method to provide fallback values: .. doctest:: @@ -232,7 +232,7 @@ even if we specify a fallback: >>> topsecret.get('CompressionLevel', '3') '9' -One more thing to be aware of is that the parser-level :meth:`get` method +One more thing to be aware of is that the parser-level :meth:`~ConfigParser.get` method provides a custom, more complex interface, maintained for backwards compatibility. When using this method, a fallback value can be provided via the ``fallback`` keyword-only argument: @@ -271,7 +271,7 @@ out. Values can also span multiple lines, as long as they are indented deeper than the first line of the value. Depending on the parser's mode, blank lines may be treated as parts of multiline values or ignored. -By default, a valid section name can be any string that does not contain '\\n' or ']'. +By default, a valid section name can be any string that does not contain '\\n'. To change this, see :attr:`ConfigParser.SECTCRE`. Configuration files may include comments, prefixed by specific @@ -481,7 +481,7 @@ historical background and it's very likely that you will want to customize some of the features. The most common way to change the way a specific config parser works is to use -the :meth:`__init__` options: +the :meth:`!__init__` options: * *defaults*, default value: ``None`` @@ -491,7 +491,7 @@ the :meth:`__init__` options: the documented default. Hint: if you want to specify default values for a specific section, use - :meth:`read_dict` before you read the actual file. + :meth:`~ConfigParser.read_dict` before you read the actual file. * *dict_type*, default value: :class:`dict` @@ -635,8 +635,8 @@ the :meth:`__init__` options: * *strict*, default value: ``True`` When set to ``True``, the parser will not allow for any section or option - duplicates while reading from a single source (using :meth:`read_file`, - :meth:`read_string` or :meth:`read_dict`). It is recommended to use strict + duplicates while reading from a single source (using :meth:`~ConfigParser.read_file`, + :meth:`~ConfigParser.read_string` or :meth:`~ConfigParser.read_dict`). It is recommended to use strict parsers in new applications. .. versionchanged:: 3.2 @@ -697,7 +697,7 @@ the :meth:`__init__` options: desirable, users may define them in a subclass or pass a dictionary where each key is a name of the converter and each value is a callable implementing said conversion. For instance, passing ``{'decimal': decimal.Decimal}`` would add - :meth:`getdecimal` on both the parser object and all section proxies. In + :meth:`!getdecimal` on both the parser object and all section proxies. In other words, it will be possible to write both ``parser_instance.getdecimal('section', 'key', fallback=0)`` and ``parser_instance['section'].getdecimal('key', 0)``. @@ -955,7 +955,7 @@ ConfigParser Objects When *converters* is given, it should be a dictionary where each key represents the name of a type converter and each value is a callable implementing the conversion from string to the desired datatype. Every - converter gets its own corresponding :meth:`get*()` method on the parser + converter gets its own corresponding :meth:`!get*()` method on the parser object and section proxies. .. versionchanged:: 3.1 @@ -1062,11 +1062,11 @@ ConfigParser Objects yielding Unicode strings (for example files opened in text mode). Optional argument *source* specifies the name of the file being read. If - not given and *f* has a :attr:`name` attribute, that is used for + not given and *f* has a :attr:`!name` attribute, that is used for *source*; the default is ``''``. .. versionadded:: 3.2 - Replaces :meth:`readfp`. + Replaces :meth:`!readfp`. .. method:: read_string(string, source='') @@ -1214,7 +1214,7 @@ ConfigParser Objects .. data:: MAX_INTERPOLATION_DEPTH - The maximum depth for recursive interpolation for :meth:`get` when the *raw* + The maximum depth for recursive interpolation for :meth:`~configparser.ConfigParser.get` when the *raw* parameter is false. This is relevant only when the default *interpolation* is used. @@ -1287,13 +1287,13 @@ Exceptions .. exception:: DuplicateSectionError - Exception raised if :meth:`add_section` is called with the name of a section + Exception raised if :meth:`~ConfigParser.add_section` is called with the name of a section that is already present or in strict parsers when a section if found more than once in a single input file, string or dictionary. .. versionadded:: 3.2 Optional ``source`` and ``lineno`` attributes and arguments to - :meth:`__init__` were added. + :meth:`!__init__` were added. .. exception:: DuplicateOptionError @@ -1345,9 +1345,9 @@ Exceptions Exception raised when errors occur attempting to parse a file. -.. versionchanged:: 3.12 - The ``filename`` attribute and :meth:`__init__` constructor argument were - removed. They have been available using the name ``source`` since 3.2. + .. versionchanged:: 3.12 + The ``filename`` attribute and :meth:`!__init__` constructor argument were + removed. They have been available using the name ``source`` since 3.2. .. rubric:: Footnotes diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index f6ebbfacfba509..aab319cbe7405e 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -106,8 +106,8 @@ Functions and classes provided: This function is a :term:`decorator` that can be used to define a factory function for :keyword:`async with` statement asynchronous context managers, - without needing to create a class or separate :meth:`__aenter__` and - :meth:`__aexit__` methods. It must be applied to an :term:`asynchronous + without needing to create a class or separate :meth:`~object.__aenter__` and + :meth:`~object.__aexit__` methods. It must be applied to an :term:`asynchronous generator` function. A simple example:: @@ -616,12 +616,12 @@ Functions and classes provided: asynchronous context managers, as well as having coroutines for cleanup logic. - The :meth:`close` method is not implemented, :meth:`aclose` must be used + The :meth:`~ExitStack.close` method is not implemented; :meth:`aclose` must be used instead. .. coroutinemethod:: enter_async_context(cm) - Similar to :meth:`enter_context` but expects an asynchronous context + Similar to :meth:`ExitStack.enter_context` but expects an asynchronous context manager. .. versionchanged:: 3.11 @@ -630,16 +630,16 @@ Functions and classes provided: .. method:: push_async_exit(exit) - Similar to :meth:`push` but expects either an asynchronous context manager + Similar to :meth:`ExitStack.push` but expects either an asynchronous context manager or a coroutine function. .. method:: push_async_callback(callback, /, *args, **kwds) - Similar to :meth:`callback` but expects a coroutine function. + Similar to :meth:`ExitStack.callback` but expects a coroutine function. .. coroutinemethod:: aclose() - Similar to :meth:`close` but properly handles awaitables. + Similar to :meth:`ExitStack.close` but properly handles awaitables. Continuing the example for :func:`asynccontextmanager`:: diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index aba398b8ee1e54..7a5589e68b3052 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -55,10 +55,11 @@ The :mod:`csv` module defines the following functions: .. function:: reader(csvfile, dialect='excel', **fmtparams) - Return a reader object which will iterate over lines in the given *csvfile*. - *csvfile* can be any object which supports the :term:`iterator` protocol and returns a - string each time its :meth:`!__next__` method is called --- :term:`file objects - ` and list objects are both suitable. If *csvfile* is a file object, + Return a :ref:`reader object ` that will process + lines from the given *csvfile*. A csvfile must be an iterable of + strings, each in the reader's defined csv format. + A csvfile is most commonly a file-like object or list. + If *csvfile* is a file object, it should be opened with ``newline=''``. [1]_ An optional *dialect* parameter can be given which is used to define a set of parameters specific to a particular CSV dialect. It may be an instance of a subclass of @@ -309,6 +310,8 @@ An example for :class:`Sniffer` use:: # ... process CSV file contents here ... +.. _csv-constants: + The :mod:`csv` module defines the following constants: .. data:: QUOTE_ALL @@ -432,8 +435,8 @@ Dialects support the following attributes: .. attribute:: Dialect.quoting Controls when quotes should be generated by the writer and recognised by the - reader. It can take on any of the :const:`QUOTE_\*` constants (see section - :ref:`csv-contents`) and defaults to :const:`QUOTE_MINIMAL`. + reader. It can take on any of the :ref:`QUOTE_\* constants ` + and defaults to :const:`QUOTE_MINIMAL`. .. attribute:: Dialect.skipinitialspace @@ -447,6 +450,8 @@ Dialects support the following attributes: When ``True``, raise exception :exc:`Error` on bad CSV input. The default is ``False``. +.. _reader-objects: + Reader Objects -------------- diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 1bab1fb9bd9464..3674b4bd97d39d 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1799,6 +1799,8 @@ Other constructor: Examples:: + .. doctest:: + >>> from datetime import time >>> time.fromisoformat('04:23:01') datetime.time(4, 23, 1) @@ -1808,7 +1810,7 @@ Other constructor: datetime.time(4, 23, 1) >>> time.fromisoformat('04:23:01.000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01,000') + >>> time.fromisoformat('04:23:01,000384') datetime.time(4, 23, 1, 384) >>> time.fromisoformat('04:23:01+04:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) @@ -1985,7 +1987,8 @@ Examples of working with a :class:`.time` object:: American EST and EDT. Special requirement for pickling: A :class:`tzinfo` subclass must have an - :meth:`__init__` method that can be called with no arguments, otherwise it can be + :meth:`~object.__init__` method that can be called with no arguments, + otherwise it can be pickled but possibly not unpickled again. This is a technical requirement that may be relaxed in the future. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 766847b971b645..cb95c61322582f 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -272,6 +272,13 @@ This module can be used with the "classic" ndbm interface or the GNU GDBM compatibility interface. On Unix, the :program:`configure` script will attempt to locate the appropriate header file to simplify building this module. +.. warning:: + + The ndbm library shipped as part of macOS has an undocumented limitation on the + size of values, which can result in corrupted database files + when storing values larger than this limit. Reading such corrupted files can + result in a hard crash (segmentation fault). + .. exception:: error Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 7e97f1a4524554..7492ae85c4ea46 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -328,26 +328,32 @@ operation is being performed, so the intermediate analysis object isn't useful: source line information (if any) is taken directly from the disassembled code object. - The *show_caches* and *adaptive* parameters work as they do in :func:`dis`. + The *adaptive* parameter works as it does in :func:`dis`. .. versionadded:: 3.4 .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + The *show_caches* parameter is deprecated and has no effect. The *cache_info* + field of each instruction is populated regardless of its value. + .. function:: findlinestarts(code) - This generator function uses the ``co_lines`` method - of the code object *code* to find the offsets which are starts of + This generator function uses the :meth:`~codeobject.co_lines` method + of the :ref:`code object ` *code* to find the offsets which + are starts of lines in the source code. They are generated as ``(offset, lineno)`` pairs. .. versionchanged:: 3.6 Line numbers can be decreasing. Before, they were always increasing. .. versionchanged:: 3.10 - The :pep:`626` ``co_lines`` method is used instead of the ``co_firstlineno`` - and ``co_lnotab`` attributes of the code object. + The :pep:`626` :meth:`~codeobject.co_lines` method is used instead of the + :attr:`~codeobject.co_firstlineno` and :attr:`~codeobject.co_lnotab` + attributes of the :ref:`code object `. .. versionchanged:: 3.13 Line numbers can be ``None`` for bytecode that does not map to source lines. @@ -481,6 +487,14 @@ details of bytecode instructions as :class:`Instruction` instances: :class:`dis.Positions` object holding the start and end locations that are covered by this instruction. + .. data::cache_info + + Information about the cache entries of this instruction, as + triplets of the form ``(name, size, data)``, where the ``name`` + and ``size`` describe the cache format and data is the contents + of the cache. ``cache_info`` is ``None`` if the instruction does not have + caches. + .. versionadded:: 3.4 .. versionchanged:: 3.11 @@ -492,8 +506,8 @@ details of bytecode instructions as :class:`Instruction` instances: Changed field ``starts_line``. Added fields ``start_offset``, ``cache_offset``, ``end_offset``, - ``baseopname``, ``baseopcode``, ``jump_target``, ``oparg``, and - ``line_number``. + ``baseopname``, ``baseopcode``, ``jump_target``, ``oparg``, + ``line_number`` and ``cache_info``. .. class:: Positions @@ -865,8 +879,8 @@ iterations of the loop. .. opcode:: RERAISE Re-raises the exception currently on top of the stack. If oparg is non-zero, - pops an additional value from the stack which is used to set ``f_lasti`` - of the current frame. + pops an additional value from the stack which is used to set + :attr:`~frame.f_lasti` of the current frame. .. versionadded:: 3.9 @@ -983,13 +997,13 @@ iterations of the loop. .. opcode:: STORE_NAME (namei) Implements ``name = STACK.pop()``. *namei* is the index of *name* in the attribute - :attr:`!co_names` of the :ref:`code object `. + :attr:`~codeobject.co_names` of the :ref:`code object `. The compiler tries to use :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. .. opcode:: DELETE_NAME (namei) - Implements ``del name``, where *namei* is the index into :attr:`!co_names` + Implements ``del name``, where *namei* is the index into :attr:`~codeobject.co_names` attribute of the :ref:`code object `. @@ -1029,7 +1043,7 @@ iterations of the loop. value = STACK.pop() obj.name = value - where *namei* is the index of name in :attr:`!co_names` of the + where *namei* is the index of name in :attr:`~codeobject.co_names` of the :ref:`code object `. .. opcode:: DELETE_ATTR (namei) @@ -1039,7 +1053,7 @@ iterations of the loop. obj = STACK.pop() del obj.name - where *namei* is the index of name into :attr:`!co_names` of the + where *namei* is the index of name into :attr:`~codeobject.co_names` of the :ref:`code object `. @@ -1201,9 +1215,10 @@ iterations of the loop. ``super(cls, self).method()``, ``super(cls, self).attr``). It pops three values from the stack (from top of stack down): - - ``self``: the first argument to the current method - - ``cls``: the class within which the current method was defined - - the global ``super`` + + * ``self``: the first argument to the current method + * ``cls``: the class within which the current method was defined + * the global ``super`` With respect to its argument, it works similarly to :opcode:`LOAD_ATTR`, except that ``namei`` is shifted left by 2 bits instead of 1. @@ -1402,7 +1417,7 @@ iterations of the loop. Pushes a reference to the object the cell contains on the stack. .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + ``i`` is no longer offset by the length of :attr:`~codeobject.co_varnames`. .. opcode:: LOAD_FROM_DICT_OR_DEREF (i) @@ -1424,7 +1439,7 @@ iterations of the loop. storage. .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + ``i`` is no longer offset by the length of :attr:`~codeobject.co_varnames`. .. opcode:: DELETE_DEREF (i) @@ -1435,7 +1450,7 @@ iterations of the loop. .. versionadded:: 3.2 .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + ``i`` is no longer offset by the length of :attr:`~codeobject.co_varnames`. .. opcode:: COPY_FREE_VARS (n) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index fa1b850c531346..8c28e4478bb70e 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1502,7 +1502,7 @@ DocTestRunner objects :attr:`failures` and :attr:`skips` attributes. The :meth:`run` and :meth:`summarize` methods return a :class:`TestResults` instance. - :class:`DocTestParser` defines the following methods: + :class:`DocTestRunner` defines the following methods: .. method:: report_start(out, test, example) diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 345b64001c1ace..d693a9bc3933b5 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -58,13 +58,18 @@ of the new API. begins with angle brackets, they are stripped off. -.. function:: parseaddr(address) +.. function:: parseaddr(address, *, strict=True) Parse address -- which should be the value of some address-containing field such as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and *email address* parts. Returns a tuple of that information, unless the parse fails, in which case a 2-tuple of ``('', '')`` is returned. + If *strict* is true, use a strict parser which rejects malformed inputs. + + .. versionchanged:: 3.13 + Add *strict* optional parameter and reject malformed inputs by default. + .. function:: formataddr(pair, charset='utf-8') @@ -82,12 +87,15 @@ of the new API. Added the *charset* option. -.. function:: getaddresses(fieldvalues) +.. function:: getaddresses(fieldvalues, *, strict=True) This method returns a list of 2-tuples of the form returned by ``parseaddr()``. *fieldvalues* is a sequence of header field values as might be returned by - :meth:`Message.get_all `. Here's a simple - example that gets all the recipients of a message:: + :meth:`Message.get_all `. + + If *strict* is true, use a strict parser which rejects malformed inputs. + + Here's a simple example that gets all the recipients of a message:: from email.utils import getaddresses @@ -97,6 +105,9 @@ of the new API. resent_ccs = msg.get_all('resent-cc', []) all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs) + .. versionchanged:: 3.13 + Add *strict* optional parameter and reject malformed inputs by default. + .. function:: parsedate(date) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 2d5ae361c3f1e3..20222bfb3611ab 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -235,6 +235,10 @@ Data Types >>> len(Color) 3 + .. attribute:: EnumType.__members__ + + Returns a mapping of every enum name to its member, including aliases + .. method:: EnumType.__reversed__(cls) Returns each member in *cls* in reverse definition order:: @@ -242,9 +246,19 @@ Data Types >>> list(reversed(Color)) [, , ] + .. method:: EnumType._add_alias_ + + Adds a new name as an alias to an existing member. Raises a + :exc:`NameError` if the name is already assigned to a different member. + + .. method:: EnumType._add_value_alias_ + + Adds a new value as an alias to an existing member. Raises a + :exc:`ValueError` if the value is already linked with a different member. + .. versionadded:: 3.11 - Before 3.11 ``enum`` used ``EnumMeta`` type, which is kept as an alias. + Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. .. class:: Enum @@ -323,7 +337,7 @@ Data Types >>> PowersOfThree.SECOND.value 9 - .. method:: Enum.__init_subclass__(cls, **kwds) + .. method:: Enum.__init_subclass__(cls, \**kwds) A *classmethod* that is used to further configure subsequent subclasses. By default, does nothing. @@ -549,7 +563,7 @@ Data Types .. method:: __invert__(self): - Returns all the flags in *type(self)* that are not in self:: + Returns all the flags in *type(self)* that are not in *self*:: >>> ~white @@ -769,37 +783,41 @@ Supported ``__dunder__`` names :attr:`~EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` items. It is only available on the class. -:meth:`~object.__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`!_value_` appropriately. Once -all the members are created it is no longer used. +:meth:`~object.__new__`, if specified, must create and return the enum members; +it is also a very good idea to set the member's :attr:`!_value_` appropriately. +Once all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- ``_name_`` -- name of the member -- ``_value_`` -- value of the member; can be set / modified in ``__new__`` - -- ``_missing_`` -- a lookup function used when a value is not found; may be - overridden -- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, - that will not be transformed into members, and will be removed from the final - class -- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent - (class attribute, removed during class creation) -- ``_generate_next_value_`` -- used to get an appropriate value for an enum - member; may be overridden +- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an + existing member. +- :attr:`~Enum._name_` -- name of the member +- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` +- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; + may be overridden +- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a + :class:`str`, that will not be transformed into members, and will be removed + from the final class +- :attr:`~Enum._order_` -- used in Python 2/3 code to ensure member order is + consistent (class attribute, removed during class creation) +- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for + an enum member; may be overridden .. note:: - For standard :class:`Enum` classes the next value chosen is the last value seen - incremented by one. + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. For :class:`Flag` classes the next value chosen will be the next highest - power-of-two, regardless of the last value seen. + power-of-two. .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` --------------- diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index cd85df8723a76b..f821776c286133 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -38,36 +38,48 @@ information on defining exceptions is available in the Python Tutorial under Exception context ----------------- -When raising a new exception while another exception -is already being handled, the new exception's -:attr:`__context__` attribute is automatically set to the handled -exception. An exception may be handled when an :keyword:`except` or -:keyword:`finally` clause, or a :keyword:`with` statement, is used. - -This implicit exception context can be -supplemented with an explicit cause by using :keyword:`!from` with -:keyword:`raise`:: - - raise new_exc from original_exc - -The expression following :keyword:`from` must be an exception or ``None``. It -will be set as :attr:`__cause__` on the raised exception. Setting -:attr:`__cause__` also implicitly sets the :attr:`__suppress_context__` -attribute to ``True``, so that using ``raise new_exc from None`` -effectively replaces the old exception with the new one for display -purposes (e.g. converting :exc:`KeyError` to :exc:`AttributeError`), while -leaving the old exception available in :attr:`__context__` for introspection -when debugging. - -The default traceback display code shows these chained exceptions in -addition to the traceback for the exception itself. An explicitly chained -exception in :attr:`__cause__` is always shown when present. An implicitly -chained exception in :attr:`__context__` is shown only if :attr:`__cause__` -is :const:`None` and :attr:`__suppress_context__` is false. - -In either case, the exception itself is always shown after any chained -exceptions so that the final line of the traceback always shows the last -exception that was raised. +.. index:: pair: exception; chaining + __cause__ (exception attribute) + __context__ (exception attribute) + __suppress_context__ (exception attribute) + +Three attributes on exception objects provide information about the context in +which the exception was raised: + +.. attribute:: BaseException.__context__ + BaseException.__cause__ + BaseException.__suppress_context__ + + When raising a new exception while another exception + is already being handled, the new exception's + :attr:`!__context__` attribute is automatically set to the handled + exception. An exception may be handled when an :keyword:`except` or + :keyword:`finally` clause, or a :keyword:`with` statement, is used. + + This implicit exception context can be + supplemented with an explicit cause by using :keyword:`!from` with + :keyword:`raise`:: + + raise new_exc from original_exc + + The expression following :keyword:`from` must be an exception or ``None``. It + will be set as :attr:`!__cause__` on the raised exception. Setting + :attr:`!__cause__` also implicitly sets the :attr:`!__suppress_context__` + attribute to ``True``, so that using ``raise new_exc from None`` + effectively replaces the old exception with the new one for display + purposes (e.g. converting :exc:`KeyError` to :exc:`AttributeError`), while + leaving the old exception available in :attr:`!__context__` for introspection + when debugging. + + The default traceback display code shows these chained exceptions in + addition to the traceback for the exception itself. An explicitly chained + exception in :attr:`!__cause__` is always shown when present. An implicitly + chained exception in :attr:`!__context__` is shown only if :attr:`!__cause__` + is :const:`None` and :attr:`!__suppress_context__` is false. + + In either case, the exception itself is always shown after any chained + exceptions so that the final line of the traceback always shows the last + exception that was raised. Inheriting from built-in exceptions @@ -126,6 +138,12 @@ The following exceptions are used mostly as base classes for other exceptions. tb = sys.exception().__traceback__ raise OtherException(...).with_traceback(tb) + .. attribute:: __traceback__ + + A writable field that holds the + :ref:`traceback object ` associated with this + exception. See also: :ref:`raise`. + .. method:: add_note(note) Add the string ``note`` to the exception's notes which appear in the standard @@ -429,9 +447,11 @@ The following exceptions are the exceptions that are usually raised. :meth:`~iterator.__next__` method to signal that there are no further items produced by the iterator. - The exception object has a single attribute :attr:`value`, which is - given as an argument when constructing the exception, and defaults - to :const:`None`. + .. attribute:: StopIteration.value + + The exception object has a single attribute :attr:`!value`, which is + given as an argument when constructing the exception, and defaults + to :const:`None`. When a :term:`generator` or :term:`coroutine` function returns, a new :exc:`StopIteration` instance is @@ -927,8 +947,10 @@ their subgroups based on the types of the contained exceptions. true for the exceptions that should be in the subgroup. The nesting structure of the current exception is preserved in the result, - as are the values of its :attr:`message`, :attr:`__traceback__`, - :attr:`__cause__`, :attr:`__context__` and :attr:`__notes__` fields. + as are the values of its :attr:`message`, + :attr:`~BaseException.__traceback__`, :attr:`~BaseException.__cause__`, + :attr:`~BaseException.__context__` and + :attr:`~BaseException.__notes__` fields. Empty nested groups are omitted from the result. The condition is checked for all exceptions in the nested exception group, @@ -954,10 +976,14 @@ their subgroups based on the types of the contained exceptions. and :meth:`split` return instances of the subclass rather than :exc:`ExceptionGroup`. - :meth:`subgroup` and :meth:`split` copy the :attr:`__traceback__`, - :attr:`__cause__`, :attr:`__context__` and :attr:`__notes__` fields from + :meth:`subgroup` and :meth:`split` copy the + :attr:`~BaseException.__traceback__`, + :attr:`~BaseException.__cause__`, :attr:`~BaseException.__context__` and + :attr:`~BaseException.__notes__` fields from the original exception group to the one returned by :meth:`derive`, so - these fields do not need to be updated by :meth:`derive`. :: + these fields do not need to be updated by :meth:`derive`. + + .. doctest:: >>> class MyGroup(ExceptionGroup): ... def derive(self, excs): @@ -983,9 +1009,9 @@ their subgroups based on the types of the contained exceptions. True - Note that :exc:`BaseExceptionGroup` defines :meth:`__new__`, so + Note that :exc:`BaseExceptionGroup` defines :meth:`~object.__new__`, so subclasses that need a different constructor signature need to - override that rather than :meth:`__init__`. For example, the following + override that rather than :meth:`~object.__init__`. For example, the following defines an exception group subclass which accepts an exit_code and and constructs the group's message from it. :: diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 509c63686f5a7f..887c3844d20faa 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -106,6 +106,10 @@ another rational number, or from a string. presentation types ``"e"``, ``"E"``, ``"f"``, ``"F"``, ``"g"``, ``"G"`` and ``"%""``. + .. versionchanged:: 3.13 + Formatting of :class:`Fraction` instances without a presentation type + now supports fill, alignment, sign handling, minimum width and grouping. + .. attribute:: numerator Numerator of the Fraction in lowest term. @@ -201,17 +205,36 @@ another rational number, or from a string. .. method:: __format__(format_spec, /) - Provides support for float-style formatting of :class:`Fraction` - instances via the :meth:`str.format` method, the :func:`format` built-in - function, or :ref:`Formatted string literals `. The - presentation types ``"e"``, ``"E"``, ``"f"``, ``"F"``, ``"g"``, ``"G"`` - and ``"%"`` are supported. For these presentation types, formatting for a - :class:`Fraction` object ``x`` follows the rules outlined for - the :class:`float` type in the :ref:`formatspec` section. + Provides support for formatting of :class:`Fraction` instances via the + :meth:`str.format` method, the :func:`format` built-in function, or + :ref:`Formatted string literals `. + + If the ``format_spec`` format specification string does not end with one + of the presentation types ``'e'``, ``'E'``, ``'f'``, ``'F'``, ``'g'``, + ``'G'`` or ``'%'`` then formatting follows the general rules for fill, + alignment, sign handling, minimum width, and grouping as described in the + :ref:`format specification mini-language `. The "alternate + form" flag ``'#'`` is supported: if present, it forces the output string + to always include an explicit denominator, even when the value being + formatted is an exact integer. The zero-fill flag ``'0'`` is not + supported. + + If the ``format_spec`` format specification string ends with one of + the presentation types ``'e'``, ``'E'``, ``'f'``, ``'F'``, ``'g'``, + ``'G'`` or ``'%'`` then formatting follows the rules outlined for the + :class:`float` type in the :ref:`formatspec` section. Here are some examples:: >>> from fractions import Fraction + >>> format(Fraction(103993, 33102), '_') + '103_993/33_102' + >>> format(Fraction(1, 7), '.^+10') + '...+1/7...' + >>> format(Fraction(3, 1), '') + '3' + >>> format(Fraction(3, 1), '#') + '3/1' >>> format(Fraction(1, 7), '.40g') '0.1428571428571428571428571428571428571429' >>> format(Fraction('1234567.855'), '_.2f') diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index b2dd32f925ef4d..4682ec9c924757 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -57,7 +57,8 @@ are always available. They are listed here in alphabetical order. .. function:: abs(x) Return the absolute value of a number. The argument may be an - integer, a floating point number, or an object implementing :meth:`__abs__`. + integer, a floating point number, or an object implementing + :meth:`~object.__abs__`. If the argument is a complex number, its magnitude is returned. @@ -235,7 +236,7 @@ are always available. They are listed here in alphabetical order. :const:`False` if not. If this returns ``True``, it is still possible that a call fails, but if it is ``False``, calling *object* will never succeed. Note that classes are callable (calling a class returns a new instance); - instances are callable if their class has a :meth:`__call__` method. + instances are callable if their class has a :meth:`~object.__call__` method. .. versionadded:: 3.2 This function was first removed in Python 3.0 and then brought back @@ -432,15 +433,18 @@ are always available. They are listed here in alphabetical order. Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object. - If the object has a method named :meth:`__dir__`, this method will be called and + If the object has a method named :meth:`~object.__dir__`, + this method will be called and must return the list of attributes. This allows objects that implement a custom - :func:`__getattr__` or :func:`__getattribute__` function to customize the way + :func:`~object.__getattr__` or :func:`~object.__getattribute__` function + to customize the way :func:`dir` reports their attributes. - If the object does not provide :meth:`__dir__`, the function tries its best to - gather information from the object's :attr:`~object.__dict__` attribute, if defined, and + If the object does not provide :meth:`~object.__dir__`, + the function tries its best to gather information from the object's + :attr:`~object.__dict__` attribute, if defined, and from its type object. The resulting list is not necessarily complete and may - be inaccurate when the object has a custom :func:`__getattr__`. + be inaccurate when the object has a custom :func:`~object.__getattr__`. The default :func:`dir` mechanism behaves differently with different types of objects, as it attempts to produce the most relevant, rather than complete, @@ -664,7 +668,7 @@ are always available. They are listed here in alphabetical order. sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" - digitpart: `digit` (["_"] `digit`)* + digitpart: `!digit` (["_"] `!digit`)* number: [`digitpart`] "." `digitpart` | `digitpart` ["."] exponent: ("e" | "E") ["+" | "-"] `digitpart` floatnumber: number [`exponent`] @@ -727,8 +731,8 @@ are always available. They are listed here in alphabetical order. A call to ``format(value, format_spec)`` is translated to ``type(value).__format__(value, format_spec)`` which bypasses the instance - dictionary when searching for the value's :meth:`__format__` method. A - :exc:`TypeError` exception is raised if the method search reaches + dictionary when searching for the value's :meth:`~object.__format__` method. + A :exc:`TypeError` exception is raised if the method search reaches :mod:`object` and the *format_spec* is non-empty, or if either the *format_spec* or the return value are not strings. @@ -792,9 +796,9 @@ are always available. They are listed here in alphabetical order. .. note:: - For objects with custom :meth:`__hash__` methods, note that :func:`hash` + For objects with custom :meth:`~object.__hash__` methods, + note that :func:`hash` truncates the return value based on the bit width of the host machine. - See :meth:`__hash__ ` for details. .. function:: help() help(request) @@ -982,7 +986,8 @@ are always available. They are listed here in alphabetical order. Return an :term:`iterator` object. The first argument is interpreted very differently depending on the presence of the second argument. Without a second argument, *object* must be a collection object which supports the - :term:`iterable` protocol (the :meth:`__iter__` method), or it must support + :term:`iterable` protocol (the :meth:`~object.__iter__` method), + or it must support the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, @@ -1500,38 +1505,44 @@ are always available. They are listed here in alphabetical order. """Get the current voltage.""" return self._voltage - The ``@property`` decorator turns the :meth:`voltage` method into a "getter" + The ``@property`` decorator turns the :meth:`!voltage` method into a "getter" for a read-only attribute with the same name, and it sets the docstring for *voltage* to "Get the current voltage." - A property object has :attr:`~property.getter`, :attr:`~property.setter`, - and :attr:`~property.deleter` methods usable as decorators that create a - copy of the property with the corresponding accessor function set to the - decorated function. This is best explained with an example:: + .. decorator:: property.getter + .. decorator:: property.setter + .. decorator:: property.deleter - class C: - def __init__(self): - self._x = None + A property object has ``getter``, ``setter``, + and ``deleter`` methods usable as decorators that create a + copy of the property with the corresponding accessor function set to the + decorated function. This is best explained with an example: - @property - def x(self): - """I'm the 'x' property.""" - return self._x + .. testcode:: - @x.setter - def x(self, value): - self._x = value + class C: + def __init__(self): + self._x = None - @x.deleter - def x(self): - del self._x + @property + def x(self): + """I'm the 'x' property.""" + return self._x - This code is exactly equivalent to the first example. Be sure to give the - additional functions the same name as the original property (``x`` in this - case.) + @x.setter + def x(self, value): + self._x = value - The returned property object also has the attributes ``fget``, ``fset``, and - ``fdel`` corresponding to the constructor arguments. + @x.deleter + def x(self): + del self._x + + This code is exactly equivalent to the first example. Be sure to give the + additional functions the same name as the original property (``x`` in this + case.) + + The returned property object also has the attributes ``fget``, ``fset``, and + ``fdel`` corresponding to the constructor arguments. .. versionchanged:: 3.5 The docstrings of property objects are now writeable. @@ -1554,7 +1565,8 @@ are always available. They are listed here in alphabetical order. representation is a string enclosed in angle brackets that contains the name of the type of the object together with additional information often including the name and address of the object. A class can control what this - function returns for its instances by defining a :meth:`__repr__` method. + function returns for its instances + by defining a :meth:`~object.__repr__` method. If :func:`sys.displayhook` is not accessible, this function will raise :exc:`RuntimeError`. @@ -1562,9 +1574,9 @@ are always available. They are listed here in alphabetical order. .. function:: reversed(seq) Return a reverse :term:`iterator`. *seq* must be an object which has - a :meth:`__reversed__` method or supports the sequence protocol (the - :meth:`__len__` method and the :meth:`~object.__getitem__` method with integer - arguments starting at ``0``). + a :meth:`~object.__reversed__` method or supports the sequence protocol (the + :meth:`~object.__len__` method and the :meth:`~object.__getitem__` method + with integer arguments starting at ``0``). .. function:: round(number, ndigits=None) @@ -1635,13 +1647,21 @@ are always available. They are listed here in alphabetical order. Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to - ``None``. Slice objects have read-only data attributes :attr:`~slice.start`, - :attr:`~slice.stop`, and :attr:`~slice.step` which merely return the argument - values (or their default). They have no other explicit functionality; - however, they are used by NumPy and other third-party packages. + ``None``. + + .. attribute:: slice.start + .. attribute:: slice.stop + .. attribute:: slice.step + + Slice objects have read-only data attributes :attr:`!start`, + :attr:`!stop`, and :attr:`!step` which merely return the argument + values (or their default). They have no other explicit functionality; + however, they are used by NumPy and other third-party packages. + Slice objects are also generated when extended indexing syntax is used. For example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an iterator. + :func:`itertools.islice` for an alternate version that returns an + :term:`iterator`. .. versionchanged:: 3.12 Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, @@ -1780,6 +1800,13 @@ are always available. They are listed here in alphabetical order. the second argument is a type, ``issubclass(type2, type)`` must be true (this is useful for classmethods). + When called directly within an ordinary method of a class, both arguments may + be omitted ("zero-argument :func:`!super`"). In this case, *type* will be the + enclosing class, and *obj* will be the first argument of the immediately + enclosing function (typically ``self``). (This means that zero-argument + :func:`!super` will not work as expected within nested functions, including + generator expressions, which implicitly create nested functions.) + There are two typical use cases for *super*. In a class hierarchy with single inheritance, *super* can be used to refer to parent classes without naming them explicitly, thus making the code more maintainable. This use @@ -1808,7 +1835,8 @@ are always available. They are listed here in alphabetical order. Note that :func:`super` is implemented as part of the binding process for explicit dotted attribute lookups such as ``super().__getitem__(name)``. - It does so by implementing its own :meth:`__getattribute__` method for searching + It does so by implementing its own :meth:`~object.__getattribute__` method + for searching classes in a predictable order that supports cooperative multiple inheritance. Accordingly, :func:`super` is undefined for implicit lookups using statements or operators such as ``super()[name]``. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 69ec1eb3ecd89d..6749a5137b446f 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -742,7 +742,7 @@ have three read-only attributes: called. :class:`partial` objects are like :class:`function` objects in that they are -callable, weak referencable, and can have attributes. There are some important +callable, weak referenceable, and can have attributes. There are some important differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes are not created automatically. Also, :class:`partial` objects defined in classes behave like static methods and do not transform into bound methods diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 331c071cda7692..82277aa52aee01 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -42,8 +42,8 @@ The :mod:`gc` module provides the following functions: With no arguments, run a full collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The number of - unreachable objects found is returned. + :exc:`ValueError` is raised if the generation number is invalid. The sum of + collected objects and uncollectable objects is returned. The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 5c79daf0f47d8e..54c84d45a59856 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -46,7 +46,10 @@ The :mod:`getpass` module provides two functions: :envvar:`USER`, :envvar:`!LNAME` and :envvar:`USERNAME`, in order, and returns the value of the first one which is set to a non-empty string. If none are set, the login name from the password database is returned on - systems which support the :mod:`pwd` module, otherwise, an exception is - raised. + systems which support the :mod:`pwd` module, otherwise, an :exc:`OSError` + is raised. In general, this function should be preferred over :func:`os.getlogin()`. + + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` were raised. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index dc6cf5533fccbe..41beac3e0c7396 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -257,7 +257,7 @@ are the methods of :class:`!NullTranslations`: .. method:: info() - Return the "protected" :attr:`_info` variable, a dictionary containing + Return a dictionary containing the metadata found in the message catalog file. @@ -296,9 +296,9 @@ are the methods of :class:`!NullTranslations`: The :class:`GNUTranslations` class ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The :mod:`gettext` module provides one additional class derived from +The :mod:`!gettext` module provides one additional class derived from :class:`NullTranslations`: :class:`GNUTranslations`. This class overrides -:meth:`_parse` to enable reading GNU :program:`gettext` format :file:`.mo` files +:meth:`!_parse` to enable reading GNU :program:`gettext` format :file:`.mo` files in both big-endian and little-endian format. :class:`GNUTranslations` parses optional metadata out of the translation @@ -306,7 +306,7 @@ catalog. It is convention with GNU :program:`gettext` to include metadata as the translation for the empty string. This metadata is in :rfc:`822`\ -style ``key: value`` pairs, and should contain the ``Project-Id-Version`` key. If the key ``Content-Type`` is found, then the ``charset`` property is used to -initialize the "protected" :attr:`_charset` instance variable, defaulting to +initialize the "protected" :attr:`!_charset` instance variable, defaulting to ``None`` if not found. If the charset encoding is specified, then all message ids and message strings read from the catalog are converted to Unicode using this encoding, else ASCII is assumed. @@ -315,7 +315,7 @@ Since message ids are read as Unicode strings too, all ``*gettext()`` methods will assume message ids as Unicode strings, not byte strings. The entire set of key/value pairs are placed into a dictionary and set as the -"protected" :attr:`_info` instance variable. +"protected" :attr:`!_info` instance variable. If the :file:`.mo` file's magic number is invalid, the major version number is unexpected, or if other problems occur while reading the file, instantiating a @@ -636,9 +636,9 @@ implementations, and valuable experience to the creation of this module: .. rubric:: Footnotes -.. [#] The default locale directory is system dependent; for example, on RedHat Linux +.. [#] The default locale directory is system dependent; for example, on Red Hat Linux it is :file:`/usr/share/locale`, but on Solaris it is :file:`/usr/lib/locale`. - The :mod:`gettext` module does not try to support these system dependent + The :mod:`!gettext` module does not try to support these system dependent defaults; instead its default is :file:`{sys.base_prefix}/share/locale` (see :data:`sys.base_prefix`). For this reason, it is always best to call :func:`bindtextdomain` with an explicit absolute path at the start of your diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index b2ca0455d3745c..43012e03c580e8 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -14,7 +14,7 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. -.. function:: new(key, msg=None, digestmod='') +.. function:: new(key, msg=None, digestmod) Return a new hmac object. *key* is a bytes or bytearray object giving the secret key. If *msg* is present, the method call ``update(msg)`` is made. @@ -27,10 +27,9 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. Parameter *msg* can be of any type supported by :mod:`hashlib`. Parameter *digestmod* can be the name of a hash algorithm. - .. deprecated-removed:: 3.4 3.8 - MD5 as implicit default digest for *digestmod* is deprecated. - The digestmod parameter is now required. Pass it as a keyword - argument to avoid awkwardness when you do not have an initial msg. + .. versionchanged:: 3.8 + The *digestmod* argument is now required. Pass it as a keyword + argument to avoid awkwardness when you do not have an initial *msg*. .. function:: digest(key, msg, digest) @@ -114,11 +113,9 @@ A hash object has the following attributes: .. versionadded:: 3.4 -.. deprecated:: 3.9 - - The undocumented attributes ``HMAC.digest_cons``, ``HMAC.inner``, and - ``HMAC.outer`` are internal implementation details and will be removed in - Python 3.10. +.. versionchanged:: 3.10 + Removed the undocumented attributes ``HMAC.digest_cons``, ``HMAC.inner``, + and ``HMAC.outer``. This module also provides the following helper function: diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index a2c1eb00d8b33d..e91972fe621a48 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -18,16 +18,17 @@ cookie value. The module formerly strictly applied the parsing rules described in the :rfc:`2109` and :rfc:`2068` specifications. It has since been discovered that -MSIE 3.0x doesn't follow the character rules outlined in those specs and also -many current day browsers and servers have relaxed parsing rules when comes to -Cookie handling. As a result, the parsing rules used are a bit less strict. +MSIE 3.0x didn't follow the character rules outlined in those specs; many +current-day browsers and servers have also relaxed parsing rules when it comes +to cookie handling. As a result, this module now uses parsing rules that are a +bit less strict than they once were. The character set, :data:`string.ascii_letters`, :data:`string.digits` and ``!#$%&'*+-.^_`|~:`` denote the set of valid characters allowed by this module -in Cookie name (as :attr:`~Morsel.key`). +in a cookie name (as :attr:`~Morsel.key`). .. versionchanged:: 3.3 - Allowed ':' as a valid Cookie name character. + Allowed ':' as a valid cookie name character. .. note:: @@ -54,9 +55,10 @@ in Cookie name (as :attr:`~Morsel.key`). .. class:: SimpleCookie([input]) - This class derives from :class:`BaseCookie` and overrides :meth:`value_decode` - and :meth:`value_encode`. SimpleCookie supports strings as cookie values. - When setting the value, SimpleCookie calls the builtin :func:`str()` to convert + This class derives from :class:`BaseCookie` and overrides :meth:`~BaseCookie.value_decode` + and :meth:`~BaseCookie.value_encode`. :class:`!SimpleCookie` supports + strings as cookie values. When setting the value, :class:`!SimpleCookie` + calls the builtin :func:`str` to convert the value to a string. Values received from HTTP are kept as strings. .. seealso:: @@ -129,17 +131,17 @@ Morsel Objects Abstract a key/value pair, which has some :rfc:`2109` attributes. Morsels are dictionary-like objects, whose set of keys is constant --- the valid - :rfc:`2109` attributes, which are - - * ``expires`` - * ``path`` - * ``comment`` - * ``domain`` - * ``max-age`` - * ``secure`` - * ``version`` - * ``httponly`` - * ``samesite`` + :rfc:`2109` attributes, which are: + + .. attribute:: expires + path + comment + domain + max-age + secure + version + httponly + samesite The attribute :attr:`httponly` specifies that the cookie is only transferred in HTTP requests, and is not accessible through JavaScript. This is intended @@ -152,7 +154,7 @@ Morsel Objects The keys are case-insensitive and their default value is ``''``. .. versionchanged:: 3.5 - :meth:`~Morsel.__eq__` now takes :attr:`~Morsel.key` and :attr:`~Morsel.value` + :meth:`!__eq__` now takes :attr:`~Morsel.key` and :attr:`~Morsel.value` into account. .. versionchanged:: 3.7 diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 6f79b222790094..64bddd23f82933 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -65,10 +65,10 @@ provides three different variants: The handler will parse the request and the headers, then call a method specific to the request type. The method name is constructed from the - request. For example, for the request method ``SPAM``, the :meth:`do_SPAM` + request. For example, for the request method ``SPAM``, the :meth:`!do_SPAM` method will be called with no arguments. All of the relevant information is stored in instance variables of the handler. Subclasses should not need to - override or extend the :meth:`__init__` method. + override or extend the :meth:`!__init__` method. :class:`BaseHTTPRequestHandler` has the following instance variables: @@ -187,13 +187,13 @@ provides three different variants: Calls :meth:`handle_one_request` once (or, if persistent connections are enabled, multiple times) to handle incoming HTTP requests. You should - never need to override it; instead, implement appropriate :meth:`do_\*` + never need to override it; instead, implement appropriate :meth:`!do_\*` methods. .. method:: handle_one_request() This method will parse and dispatch the request to the appropriate - :meth:`do_\*` method. You should never need to override it. + :meth:`!do_\*` method. You should never need to override it. .. method:: handle_expect_100() diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 1df7d8d772a274..cc4a0da92da60a 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -171,16 +171,18 @@ group. Read `the setuptools docs `_ for more information on entry points, their definition, and usage. -*Compatibility Note* - -The "selectable" entry points were introduced in ``importlib_metadata`` -3.6 and Python 3.10. Prior to those changes, ``entry_points`` accepted -no parameters and always returned a dictionary of entry points, keyed -by group. With ``importlib_metadata`` 5.0 and Python 3.12, -``entry_points`` always returns an ``EntryPoints`` object. See -`backports.entry_points_selectable `_ -for compatibility options. - +.. versionchanged:: 3.12 + The "selectable" entry points were introduced in ``importlib_metadata`` + 3.6 and Python 3.10. Prior to those changes, ``entry_points`` accepted + no parameters and always returned a dictionary of entry points, keyed + by group. With ``importlib_metadata`` 5.0 and Python 3.12, + ``entry_points`` always returns an ``EntryPoints`` object. See + `backports.entry_points_selectable `_ + for compatibility options. + +.. versionchanged:: 3.13 + ``EntryPoint`` objects no longer present a tuple-like interface + (:meth:`~object.__getitem__`). .. _metadata: @@ -342,9 +344,17 @@ instance:: >>> dist.metadata['License'] # doctest: +SKIP 'MIT' +For editable packages, an origin property may present :pep:`610` +metadata:: + + >>> dist.origin.url + 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' + The full set of available metadata is not described here. See the `Core metadata specifications `_ for additional details. +.. versionadded:: 3.13 + The ``.origin`` property was added. Distribution Discovery ====================== diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index fc954724bb72fe..2402bc5cd3ee2c 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1145,7 +1145,7 @@ find and load modules. .. versionadded:: 3.4 -.. class:: NamespaceLoader(name, path, path_finder): +.. class:: NamespaceLoader(name, path, path_finder) A concrete implementation of :class:`importlib.abc.InspectLoader` for namespace packages. This is an alias for a private class and is only made diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index b463c0b6d0e402..f8b3e39c4f54f0 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1,6 +1,11 @@ :mod:`inspect` --- Inspect live objects ======================================= +.. testsetup:: * + + import inspect + from inspect import * + .. module:: inspect :synopsis: Extract information and source code from live objects. @@ -268,7 +273,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): :func:`getmembers` will only return class attributes defined in the metaclass when the argument is a class and those attributes have been - listed in the metaclass' custom :meth:`__dir__`. + listed in the metaclass' custom :meth:`~object.__dir__`. .. function:: getmembers_static(object[, predicate]) @@ -387,7 +392,11 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object can be used in :keyword:`await` expression. Can also be used to distinguish generator-based coroutines from regular - generators:: + generators: + + .. testcode:: + + import types def gen(): yield @@ -404,13 +413,15 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: isasyncgenfunction(object) Return ``True`` if the object is an :term:`asynchronous generator` function, - for example:: + for example: + + .. doctest:: - >>> async def agen(): - ... yield 1 - ... - >>> inspect.isasyncgenfunction(agen) - True + >>> async def agen(): + ... yield 1 + ... + >>> inspect.isasyncgenfunction(agen) + True .. versionadded:: 3.6 @@ -476,12 +487,13 @@ attributes (see :ref:`import-mod-attrs` for module attributes): has a :meth:`~object.__get__` method but not a :meth:`~object.__set__` method, but beyond that the set of attributes varies. A :attr:`~definition.__name__` attribute is usually - sensible, and :attr:`__doc__` often is. + sensible, and :attr:`!__doc__` often is. Methods implemented via descriptors that also pass one of the other tests return ``False`` from the :func:`ismethoddescriptor` test, simply because the other tests promise more -- you can, e.g., count on having the - :attr:`__func__` attribute (etc) when an object passes :func:`ismethod`. + :attr:`~method.__func__` attribute (etc) when an object passes + :func:`ismethod`. .. function:: isdatadescriptor(object) @@ -492,7 +504,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Examples are properties (defined in Python), getsets, and members. The latter two are defined in C and there are more specific tests available for those types, which is robust across Python implementations. Typically, data - descriptors will also have :attr:`~definition.__name__` and :attr:`__doc__` attributes + descriptors will also have :attr:`~definition.__name__` and :attr:`!__doc__` attributes (properties, getsets, and members have both of these attributes), but this is not guaranteed. @@ -614,13 +626,16 @@ Introspecting callables with the Signature object .. versionadded:: 3.3 -The Signature object represents the call signature of a callable object and its -return annotation. To retrieve a Signature object, use the :func:`signature` +The :class:`Signature` object represents the call signature of a callable object +and its return annotation. To retrieve a :class:`!Signature` object, +use the :func:`!signature` function. .. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) - Return a :class:`Signature` object for the given ``callable``:: + Return a :class:`Signature` object for the given *callable*: + + .. doctest:: >>> from inspect import signature >>> def foo(a, *, b:int, **kwargs): @@ -629,10 +644,10 @@ function. >>> sig = signature(foo) >>> str(sig) - '(a, *, b:int, **kwargs)' + '(a, *, b: int, **kwargs)' >>> str(sig.parameters['b']) - 'b:int' + 'b: int' >>> sig.parameters['b'].annotation @@ -646,29 +661,30 @@ function. For objects defined in modules using stringized annotations (``from __future__ import annotations``), :func:`signature` will attempt to automatically un-stringize the annotations using - :func:`inspect.get_annotations()`. The - ``global``, ``locals``, and ``eval_str`` parameters are passed - into :func:`inspect.get_annotations()` when resolving the - annotations; see the documentation for :func:`inspect.get_annotations()` + :func:`get_annotations`. The + *globals*, *locals*, and *eval_str* parameters are passed + into :func:`get_annotations` when resolving the + annotations; see the documentation for :func:`get_annotations` for instructions on how to use these parameters. Raises :exc:`ValueError` if no signature can be provided, and :exc:`TypeError` if that type of object is not supported. Also, - if the annotations are stringized, and ``eval_str`` is not false, - the ``eval()`` call(s) to un-stringize the annotations could - potentially raise any kind of exception. + if the annotations are stringized, and *eval_str* is not false, + the ``eval()`` call(s) to un-stringize the annotations in :func:`get_annotations` + could potentially raise any kind of exception. A slash(/) in the signature of a function denotes that the parameters prior to it are positional-only. For more info, see :ref:`the FAQ entry on positional-only parameters `. - .. versionadded:: 3.5 - ``follow_wrapped`` parameter. Pass ``False`` to get a signature of - ``callable`` specifically (``callable.__wrapped__`` will not be used to + .. versionchanged:: 3.5 + The *follow_wrapped* parameter was added. + Pass ``False`` to get a signature of + *callable* specifically (``callable.__wrapped__`` will not be used to unwrap decorated callables.) - .. versionadded:: 3.10 - ``globals``, ``locals``, and ``eval_str`` parameters. + .. versionchanged:: 3.10 + The *globals*, *locals*, and *eval_str* parameters were added. .. note:: @@ -679,7 +695,8 @@ function. .. class:: Signature(parameters=None, *, return_annotation=Signature.empty) - A Signature object represents the call signature of a function and its return + A :class:`!Signature` object represents the call signature of a function + and its return annotation. For each parameter accepted by the function it stores a :class:`Parameter` object in its :attr:`parameters` collection. @@ -689,14 +706,14 @@ function. positional-only first, then positional-or-keyword, and that parameters with defaults follow parameters without defaults. - The optional *return_annotation* argument, can be an arbitrary Python object, - is the "return" annotation of the callable. + The optional *return_annotation* argument can be an arbitrary Python object. + It represents the "return" annotation of the callable. - Signature objects are *immutable*. Use :meth:`Signature.replace` or + :class:`!Signature` objects are *immutable*. Use :meth:`Signature.replace` or :func:`copy.replace` to make a modified copy. .. versionchanged:: 3.5 - Signature objects are picklable and :term:`hashable`. + :class:`!Signature` objects are now picklable and :term:`hashable`. .. attribute:: Signature.empty @@ -733,13 +750,15 @@ function. .. method:: Signature.replace(*[, parameters][, return_annotation]) - Create a new Signature instance based on the instance :meth:`replace` was invoked - on. It is possible to pass different ``parameters`` and/or - ``return_annotation`` to override the corresponding properties of the base - signature. To remove return_annotation from the copied Signature, pass in + Create a new :class:`Signature` instance based on the instance + :meth:`replace` was invoked on. + It is possible to pass different *parameters* and/or + *return_annotation* to override the corresponding properties of the base + signature. To remove ``return_annotation`` from the copied + :class:`!Signature`, pass in :attr:`Signature.empty`. - :: + .. doctest:: >>> def test(a, b): ... pass @@ -749,38 +768,50 @@ function. >>> str(new_sig) "(a, b) -> 'new return anno'" - Signature objects are also supported by generic function + :class:`Signature` objects are also supported by the generic function :func:`copy.replace`. - .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globalns=None, localns=None) + .. method:: format(*, max_width=None) + + Create a string representation of the :class:`Signature` object. + + If *max_width* is passed, the method will attempt to fit + the signature into lines of at most *max_width* characters. + If the signature is longer than *max_width*, + all parameters will be on separate lines. + + .. versionadded:: 3.13 + + .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) Return a :class:`Signature` (or its subclass) object for a given callable - ``obj``. Pass ``follow_wrapped=False`` to get a signature of ``obj`` - without unwrapping its ``__wrapped__`` chain. ``globalns`` and - ``localns`` will be used as the namespaces when resolving annotations. + *obj*. + + This method simplifies subclassing of :class:`Signature`: - This method simplifies subclassing of :class:`Signature`:: + .. testcode:: - class MySignature(Signature): - pass - sig = MySignature.from_callable(min) - assert isinstance(sig, MySignature) + class MySignature(Signature): + pass + sig = MySignature.from_callable(sum) + assert isinstance(sig, MySignature) Its behavior is otherwise identical to that of :func:`signature`. .. versionadded:: 3.5 - .. versionadded:: 3.10 - ``globalns`` and ``localns`` parameters. + .. versionchanged:: 3.10 + The *globals*, *locals*, and *eval_str* parameters were added. .. class:: Parameter(name, kind, *, default=Parameter.empty, annotation=Parameter.empty) - Parameter objects are *immutable*. Instead of modifying a Parameter object, + :class:`!Parameter` objects are *immutable*. + Instead of modifying a :class:`!Parameter` object, you can use :meth:`Parameter.replace` or :func:`copy.replace` to create a modified copy. .. versionchanged:: 3.5 - Parameter objects are picklable and :term:`hashable`. + Parameter objects are now picklable and :term:`hashable`. .. attribute:: Parameter.empty @@ -799,7 +830,7 @@ function. expressions. .. versionchanged:: 3.6 - These parameter names are exposed by this module as names like + These parameter names are now exposed by this module as names like ``implicit0``. .. attribute:: Parameter.default @@ -849,7 +880,9 @@ function. | | definition. | +------------------------+----------------------------------------------+ - Example: print all keyword-only arguments without default values:: + Example: print all keyword-only arguments without default values: + + .. doctest:: >>> def foo(a, b, *, c, d=10): ... pass @@ -863,11 +896,13 @@ function. .. attribute:: Parameter.kind.description - Describes a enum value of Parameter.kind. + Describes a enum value of :attr:`Parameter.kind`. .. versionadded:: 3.8 - Example: print all descriptions of arguments:: + Example: print all descriptions of arguments: + + .. doctest:: >>> def foo(a, b, *, c, d=10): ... pass @@ -882,12 +917,12 @@ function. .. method:: Parameter.replace(*[, name][, kind][, default][, annotation]) - Create a new Parameter instance based on the instance replaced was invoked - on. To override a :class:`Parameter` attribute, pass the corresponding + Create a new :class:`Parameter` instance based on the instance replaced was invoked + on. To override a :class:`!Parameter` attribute, pass the corresponding argument. To remove a default value or/and an annotation from a - Parameter, pass :attr:`Parameter.empty`. + :class:`!Parameter`, pass :attr:`Parameter.empty`. - :: + .. doctest:: >>> from inspect import Parameter >>> param = Parameter('foo', Parameter.KEYWORD_ONLY, default=42) @@ -898,12 +933,13 @@ function. 'foo=42' >>> str(param.replace(default=Parameter.empty, annotation='spam')) - "foo:'spam'" + "foo: 'spam'" - Parameter objects are also supported by generic function :func:`copy.replace`. + :class:`Parameter` objects are also supported by the generic function + :func:`copy.replace`. .. versionchanged:: 3.4 - In Python 3.3 Parameter objects were allowed to have ``name`` set + In Python 3.3 :class:`Parameter` objects were allowed to have ``name`` set to ``None`` if their ``kind`` was set to ``POSITIONAL_ONLY``. This is no longer permitted. @@ -956,18 +992,20 @@ function. For variable-keyword arguments (``**kwargs``) the default is an empty dict. - :: + .. doctest:: - >>> def foo(a, b='ham', *args): pass - >>> ba = inspect.signature(foo).bind('spam') - >>> ba.apply_defaults() - >>> ba.arguments - {'a': 'spam', 'b': 'ham', 'args': ()} + >>> def foo(a, b='ham', *args): pass + >>> ba = inspect.signature(foo).bind('spam') + >>> ba.apply_defaults() + >>> ba.arguments + {'a': 'spam', 'b': 'ham', 'args': ()} .. versionadded:: 3.5 The :attr:`args` and :attr:`kwargs` properties can be used to invoke - functions:: + functions: + + .. testcode:: def test(a, *, b): ... @@ -1086,20 +1124,22 @@ Classes and functions ``**`` arguments, if any) to their values from *args* and *kwds*. In case of invoking *func* incorrectly, i.e. whenever ``func(*args, **kwds)`` would raise an exception because of incompatible signature, an exception of the same type - and the same or similar message is raised. For example:: - - >>> from inspect import getcallargs - >>> def f(a, b=1, *pos, **named): - ... pass - ... - >>> getcallargs(f, 1, 2, 3) == {'a': 1, 'named': {}, 'b': 2, 'pos': (3,)} - True - >>> getcallargs(f, a=2, x=4) == {'a': 2, 'named': {'x': 4}, 'b': 1, 'pos': ()} - True - >>> getcallargs(f) - Traceback (most recent call last): - ... - TypeError: f() missing 1 required positional argument: 'a' + and the same or similar message is raised. For example: + + .. doctest:: + + >>> from inspect import getcallargs + >>> def f(a, b=1, *pos, **named): + ... pass + ... + >>> getcallargs(f, 1, 2, 3) == {'a': 1, 'named': {}, 'b': 2, 'pos': (3,)} + True + >>> getcallargs(f, a=2, x=4) == {'a': 2, 'named': {'x': 4}, 'b': 1, 'pos': ()} + True + >>> getcallargs(f) + Traceback (most recent call last): + ... + TypeError: f() missing 1 required positional argument: 'a' .. versionadded:: 3.2 @@ -1185,9 +1225,10 @@ Classes and functions * If ``obj`` is a class, ``globals`` defaults to ``sys.modules[obj.__module__].__dict__`` and ``locals`` defaults to the ``obj`` class namespace. - * If ``obj`` is a callable, ``globals`` defaults to ``obj.__globals__``, + * If ``obj`` is a callable, ``globals`` defaults to + :attr:`obj.__globals__ `, although if ``obj`` is a wrapped function (using - ``functools.update_wrapper()``) it is first unwrapped. + :func:`functools.update_wrapper`) it is first unwrapped. Calling ``get_annotations`` is best practice for accessing the annotations dict of any object. See :ref:`annotations-howto` for @@ -1401,7 +1442,8 @@ Fetching attributes statically Both :func:`getattr` and :func:`hasattr` can trigger code execution when fetching or checking for the existence of attributes. Descriptors, like -properties, will be invoked and :meth:`__getattr__` and :meth:`__getattribute__` +properties, will be invoked and :meth:`~object.__getattr__` and +:meth:`~object.__getattribute__` may be called. For cases where you want passive introspection, like documentation tools, this @@ -1411,7 +1453,8 @@ but avoids executing code when it fetches attributes. .. function:: getattr_static(obj, attr, default=None) Retrieve attributes without triggering dynamic lookup via the - descriptor protocol, :meth:`__getattr__` or :meth:`__getattribute__`. + descriptor protocol, :meth:`~object.__getattr__` + or :meth:`~object.__getattribute__`. Note: this function may not be able to retrieve all attributes that getattr can fetch (like dynamically created attributes) @@ -1554,8 +1597,8 @@ updated as expected: Code Objects Bit Flags ---------------------- -Python code objects have a ``co_flags`` attribute, which is a bitmap of -the following flags: +Python code objects have a :attr:`~codeobject.co_flags` attribute, +which is a bitmap of the following flags: .. data:: CO_OPTIMIZED @@ -1563,8 +1606,8 @@ the following flags: .. data:: CO_NEWLOCALS - If set, a new dict will be created for the frame's ``f_locals`` when - the code object is executed. + If set, a new dict will be created for the frame's :attr:`~frame.f_locals` + when the code object is executed. .. data:: CO_VARARGS diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index ebb4ebcfa7618a..5c8cc982a89a2c 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -164,11 +164,14 @@ loops that truncate the stream. Added the optional *initial* parameter. -.. function:: batched(iterable, n) +.. function:: batched(iterable, n, *, strict=False) Batch data from the *iterable* into tuples of length *n*. The last batch may be shorter than *n*. + If *strict* is true, will raise a :exc:`ValueError` if the final + batch is shorter than *n*. + Loops over the input iterable and accumulates data into tuples up to size *n*. The input is consumed lazily, just enough to fill a batch. The result is yielded as soon as the batch is full or when the input @@ -190,16 +193,21 @@ loops that truncate the stream. Roughly equivalent to:: - def batched(iterable, n): + def batched(iterable, n, *, strict=False): # batched('ABCDEFG', 3) --> ABC DEF G if n < 1: raise ValueError('n must be at least one') it = iter(iterable) while batch := tuple(islice(it, n)): + if strict and len(batch) != n: + raise ValueError('batched(): incomplete batch') yield batch .. versionadded:: 3.12 + .. versionchanged:: 3.13 + Added the *strict* option. + .. function:: chain(*iterables) @@ -795,11 +803,11 @@ which incur interpreter overhead. import random def take(n, iterable): - "Return first n items of the iterable as a list" + "Return first n items of the iterable as a list." return list(islice(iterable, n)) def prepend(value, iterable): - "Prepend a single value in front of an iterable" + "Prepend a single value in front of an iterable." # prepend(1, [2, 3, 4]) --> 1 2 3 4 return chain([value], iterable) @@ -817,15 +825,15 @@ which incur interpreter overhead. return starmap(func, repeat(args, times)) def flatten(list_of_lists): - "Flatten one level of nesting" + "Flatten one level of nesting." return chain.from_iterable(list_of_lists) def ncycles(iterable, n): - "Returns the sequence elements n times" + "Returns the sequence elements n times." return chain.from_iterable(repeat(tuple(iterable), n)) def tail(n, iterable): - "Return an iterator over the last n items" + "Return an iterator over the last n items." # tail(3, 'ABCDEFG') --> E F G return iter(collections.deque(iterable, maxlen=n)) @@ -840,7 +848,7 @@ which incur interpreter overhead. next(islice(iterator, n, n), None) def nth(iterable, n, default=None): - "Returns the nth item or a default value" + "Returns the nth item or a default value." return next(islice(iterable, n, None), default) def quantify(iterable, pred=bool): @@ -848,7 +856,7 @@ which incur interpreter overhead. return sum(map(pred, iterable)) def all_equal(iterable): - "Returns True if all the elements are equal to each other" + "Returns True if all the elements are equal to each other." g = groupby(iterable) return next(g, True) and not next(g, False) @@ -865,6 +873,30 @@ which incur interpreter overhead. # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x return next(filter(pred, iterable), default) + def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBcCAD', str.casefold) --> A B c D + seen = set() + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen.add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen.add(k) + yield element + + def unique_justseen(iterable, key=None): + "List unique elements, preserving order. Remember only the element just seen." + # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B + # unique_justseen('ABBcCAD', str.casefold) --> A B c A D + if key is None: + return map(operator.itemgetter(0), groupby(iterable)) + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') --> 0 1 4 7 @@ -885,45 +917,8 @@ which incur interpreter overhead. except ValueError: pass - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Examples: - iter_except(functools.partial(heappop, h), IndexError) # priority queue iterator - iter_except(d.popitem, KeyError) # non-blocking dict iterator - iter_except(d.popleft, IndexError) # non-blocking deque iterator - iter_except(q.get_nowait, Queue.Empty) # loop over a producer Queue - iter_except(s.pop, KeyError) # non-blocking set iterator - - """ - try: - if first is not None: - yield first() # For database APIs needing an initial cast to db.first() - while True: - yield func() - except exception: - pass - - def grouper(iterable, n, *, incomplete='fill', fillvalue=None): - "Collect data into non-overlapping fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx - # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError - # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF - args = [iter(iterable)] * n - if incomplete == 'fill': - return zip_longest(*args, fillvalue=fillvalue) - if incomplete == 'strict': - return zip(*args, strict=True) - if incomplete == 'ignore': - return zip(*args) - else: - raise ValueError('Expected fill, strict, or ignore') - def sliding_window(iterable, n): + "Collect data into overlapping fixed-length chunks or blocks." # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG it = iter(iterable) window = collections.deque(islice(it, n-1), maxlen=n) @@ -931,8 +926,25 @@ which incur interpreter overhead. window.append(x) yield tuple(window) + def grouper(iterable, n, *, incomplete='fill', fillvalue=None): + "Collect data into non-overlapping fixed-length chunks or blocks." + # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx + # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError + # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF + args = [iter(iterable)] * n + match incomplete: + case 'fill': + return zip_longest(*args, fillvalue=fillvalue) + case 'strict': + return zip(*args, strict=True) + case 'ignore': + return zip(*args) + case _: + raise ValueError('Expected fill, strict, or ignore') + def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + "Visit input iterables in a cycle until each is exhausted." + # roundrobin('ABC', 'D', 'EF') --> A D E B F C # Recipe credited to George Sakkis num_active = len(iterables) nexts = cycle(iter(it).__next__ for it in iterables) @@ -955,11 +967,43 @@ which incur interpreter overhead. return filterfalse(pred, t1), filter(pred, t2) def subslices(seq): - "Return all contiguous non-empty subslices of a sequence" + "Return all contiguous non-empty subslices of a sequence." # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) + def iter_except(func, exception, first=None): + """ Call a function repeatedly until an exception is raised. + + Converts a call-until-exception interface to an iterator interface. + Like builtins.iter(func, sentinel) but uses an exception instead + of a sentinel to end the loop. + + Priority queue iterator: + iter_except(functools.partial(heappop, h), IndexError) + + Non-blocking dictionary iterator: + iter_except(d.popitem, KeyError) + + Non-blocking deque iterator: + iter_except(d.popleft, IndexError) + + Non-blocking iterator over a producer Queue: + iter_except(q.get_nowait, Queue.Empty) + + Non-blocking set iterator: + iter_except(s.pop, KeyError) + + """ + try: + if first is not None: + # For database APIs needing an initial call to db.first() + yield first() + while True: + yield func() + except exception: + pass + def before_and_after(predicate, it): """ Variant of takewhile() that allows complete access to the remainder of the iterator. @@ -971,12 +1015,12 @@ which incur interpreter overhead. >>> ''.join(remainder) # takewhile() would lose the 'd' 'dEfGhI' - Note that the first iterator must be fully - consumed before the second iterator can - generate valid results. + Note that the true iterator must be fully consumed + before the remainder iterator can generate valid results. """ it = iter(it) transition = [] + def true_iterator(): for elem in it: if predicate(elem): @@ -984,39 +1028,8 @@ which incur interpreter overhead. else: transition.append(elem) return - def remainder_iterator(): - yield from transition - yield from it - return true_iterator(), remainder_iterator() - def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.lower) --> A B c D - seen = set() - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen.add(element) - yield element - # For order preserving deduplication, - # a faster but non-lazy solution is: - # yield from dict.fromkeys(iterable) - else: - for element in iterable: - k = key(element) - if k not in seen: - seen.add(k) - yield element - # For use cases that allow the last matching element to be returned, - # a faster but non-lazy solution is: - # t1, t2 = tee(iterable) - # yield from dict(zip(map(key, t1), t2)).values() - - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.lower) --> A B c A D - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + return true_iterator(), chain(transition, it) The following recipes have a more mathematical flavor: @@ -1033,10 +1046,15 @@ The following recipes have a more mathematical flavor: # sum_of_squares([10, 20, 30]) -> 1400 return math.sumprod(*tee(it)) - def transpose(it): - "Swap the rows and columns of the input." + def reshape(matrix, cols): + "Reshape a 2-D matrix to have a given number of columns." + # reshape([(0, 1), (2, 3), (4, 5)], 3) --> (0, 1, 2), (3, 4, 5) + return batched(chain.from_iterable(matrix), cols, strict=True) + + def transpose(matrix): + "Swap the rows and columns of a 2-D matrix." # transpose([(1, 2, 3), (11, 22, 33)]) --> (1, 11) (2, 22) (3, 33) - return zip(*it, strict=True) + return zip(*matrix, strict=True) def matmul(m1, m2): "Multiply two matrices." @@ -1127,23 +1145,13 @@ The following recipes have a more mathematical flavor: if n > 1: yield n - def nth_combination(iterable, r, index): - "Equivalent to list(combinations(iterable, r))[index]" - pool = tuple(iterable) - n = len(pool) - c = math.comb(n, r) - if index < 0: - index += c - if index < 0 or index >= c: - raise IndexError - result = [] - while r: - c, n, r = c*r//n, n-1, r-1 - while index >= c: - index -= c - c, n = c*(n-r)//n, n-1 - result.append(pool[-1-n]) - return tuple(result) + def totient(n): + "Count of natural numbers up to n that are coprime to n." + # https://mathworld.wolfram.com/TotientFunction.html + # totient(12) --> 4 because len([1, 5, 7, 11]) == 4 + for p in unique_justseen(factor(n)): + n = n // p * (p - 1) + return n .. doctest:: @@ -1261,6 +1269,26 @@ The following recipes have a more mathematical flavor: >>> sum_of_squares([10, 20, 30]) 1400 + >>> list(reshape([(0, 1), (2, 3), (4, 5)], 3)) + [(0, 1, 2), (3, 4, 5)] + >>> M = [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] + >>> list(reshape(M, 1)) + [(0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,), (10,), (11,)] + >>> list(reshape(M, 2)) + [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11)] + >>> list(reshape(M, 3)) + [(0, 1, 2), (3, 4, 5), (6, 7, 8), (9, 10, 11)] + >>> list(reshape(M, 4)) + [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] + >>> list(reshape(M, 5)) + Traceback (most recent call last): + ... + ValueError: batched(): incomplete batch + >>> list(reshape(M, 6)) + [(0, 1, 2, 3, 4, 5), (6, 7, 8, 9, 10, 11)] + >>> list(reshape(M, 12)) + [(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)] + >>> list(transpose([(1, 2, 3), (11, 22, 33)])) [(1, 11), (2, 22), (3, 33)] @@ -1428,6 +1456,25 @@ The following recipes have a more mathematical flavor: >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True + >>> totient(0) # https://www.wolframalpha.com/input?i=totient+0 + 0 + >>> first_totients = [1, 1, 2, 2, 4, 2, 6, 4, 6, 4, 10, 4, 12, 6, 8, 8, 16, 6, + ... 18, 8, 12, 10, 22, 8, 20, 12, 18, 12, 28, 8, 30, 16, 20, 16, 24, 12, 36, 18, + ... 24, 16, 40, 12, 42, 20, 24, 22, 46, 16, 42, 20, 32, 24, 52, 18, 40, 24, 36, + ... 28, 58, 16, 60, 30, 36, 32, 48, 20, 66, 32, 44] # https://oeis.org/A000010 + ... + >>> list(map(totient, range(1, 70))) == first_totients + True + >>> reference_totient = lambda n: sum(math.gcd(t, n) == 1 for t in range(1, n+1)) + >>> all(totient(n) == reference_totient(n) for n in range(1000)) + True + >>> totient(128_884_753_939) == 128_884_753_938 # large prime + True + >>> totient(999953 * 999983) == 999952 * 999982 # large semiprime + True + >>> totient(6 ** 20) == 1 * 2**19 * 2 * 3**19 # repeated primes + True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] @@ -1517,16 +1564,16 @@ The following recipes have a more mathematical flavor: >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) + >>> list(unique_everseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBcCAD', str.lower)) + >>> list(unique_everseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'D'] >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) + >>> list(unique_justseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'A', 'D'] - >>> list(unique_justseen('ABBcCAD', str.lower)) + >>> list(unique_justseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'A', 'D'] >>> d = dict(a=1, b=2, c=3) @@ -1549,20 +1596,6 @@ The following recipes have a more mathematical flavor: >>> first_true('ABC0DEF1', '9', str.isdigit) '0' - >>> population = 'ABCDEFGH' - >>> for r in range(len(population) + 1): - ... seq = list(combinations(population, r)) - ... for i in range(len(seq)): - ... assert nth_combination(population, r, i) == seq[i] - ... for i in range(-len(seq), 0): - ... assert nth_combination(population, r, i) == seq[i] - - >>> iterable = 'abcde' - >>> r = 3 - >>> combos = list(combinations(iterable, r)) - >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) - True - .. testcode:: :hide: @@ -1589,6 +1622,24 @@ The following recipes have a more mathematical flavor: for (a, _), (b, c) in pairwise(pairwise(iterable)): yield a, b, c + def nth_combination(iterable, r, index): + "Equivalent to list(combinations(iterable, r))[index]" + pool = tuple(iterable) + n = len(pool) + c = math.comb(n, r) + if index < 0: + index += c + if index < 0 or index >= c: + raise IndexError + result = [] + while r: + c, n, r = c*r//n, n-1, r-1 + while index >= c: + index -= c + c, n = c*(n-r)//n, n-1 + result.append(pool[-1-n]) + return tuple(result) + .. doctest:: :hide: @@ -1604,3 +1655,17 @@ The following recipes have a more mathematical flavor: >>> list(triplewise('ABCDEFG')) [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] + + >>> population = 'ABCDEFGH' + >>> for r in range(len(population) + 1): + ... seq = list(combinations(population, r)) + ... for i in range(len(seq)): + ... assert nth_combination(population, r, i) == seq[i] + ... for i in range(-len(seq), 0): + ... assert nth_combination(population, r, i) == seq[i] + + >>> iterable = 'abcde' + >>> r = 3 + >>> combos = list(combinations(iterable, r)) + >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) + True diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 0d48892fcdab97..a7201199191215 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -309,7 +309,7 @@ The :mod:`locale` module defines the following exception and functions: .. function:: getlocale(category=LC_CTYPE) Returns the current setting for the given locale category as sequence containing - *language code*, *encoding*. *category* may be one of the :const:`LC_\*` values + *language code*, *encoding*. *category* may be one of the :const:`!LC_\*` values except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 85a53e6aa7a78b..85a68cb11ee22c 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -93,8 +93,8 @@ in :mod:`logging` itself) and defining handlers which are declared either in :param fname: A filename, or a file-like object, or an instance derived from :class:`~configparser.RawConfigParser`. If a - ``RawConfigParser``-derived instance is passed, it is used as - is. Otherwise, a :class:`~configparser.Configparser` is + :class:`!RawConfigParser`-derived instance is passed, it is used as + is. Otherwise, a :class:`~configparser.ConfigParser` is instantiated, and the configuration read by it from the object passed in ``fname``. If that has a :meth:`readline` method, it is assumed to be a file-like object and read using @@ -103,7 +103,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in :meth:`~configparser.ConfigParser.read`. - :param defaults: Defaults to be passed to the ConfigParser can be specified + :param defaults: Defaults to be passed to the :class:`!ConfigParser` can be specified in this argument. :param disable_existing_loggers: If specified as ``False``, loggers which diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 05ffaf6c9b336e..fa5b273093f583 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -13,8 +13,8 @@ This module defines two classes, :class:`Mailbox` and :class:`Message`, for accessing and manipulating on-disk mailboxes and the messages they contain. -:class:`Mailbox` offers a dictionary-like mapping from keys to messages. -:class:`Message` extends the :mod:`email.message` module's +:class:`!Mailbox` offers a dictionary-like mapping from keys to messages. +:class:`!Message` extends the :mod:`email.message` module's :class:`~email.message.Message` class with format-specific state and behavior. Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. @@ -27,37 +27,38 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-objects: -:class:`Mailbox` objects ------------------------- +:class:`!Mailbox` objects +------------------------- .. class:: Mailbox A mailbox, which may be inspected and modified. - The :class:`Mailbox` class defines an interface and is not intended to be + The :class:`!Mailbox` class defines an interface and is not intended to be instantiated. Instead, format-specific subclasses should inherit from - :class:`Mailbox` and your code should instantiate a particular subclass. + :class:`!Mailbox` and your code should instantiate a particular subclass. - The :class:`Mailbox` interface is dictionary-like, with small keys - corresponding to messages. Keys are issued by the :class:`Mailbox` instance - with which they will be used and are only meaningful to that :class:`Mailbox` + The :class:`!Mailbox` interface is dictionary-like, with small keys + corresponding to messages. Keys are issued by the :class:`!Mailbox` instance + with which they will be used and are only meaningful to that :class:`!Mailbox` instance. A key continues to identify a message even if the corresponding message is modified, such as by replacing it with another message. - Messages may be added to a :class:`Mailbox` instance using the set-like + Messages may be added to a :class:`!Mailbox` instance using the set-like method :meth:`add` and removed using a ``del`` statement or the set-like methods :meth:`remove` and :meth:`discard`. - :class:`Mailbox` interface semantics differ from dictionary semantics in some + :class:`!Mailbox` interface semantics differ from dictionary semantics in some noteworthy ways. Each time a message is requested, a new representation (typically a :class:`Message` instance) is generated based upon the current state of the mailbox. Similarly, when a message is added to a - :class:`Mailbox` instance, the provided message representation's contents are + :class:`!Mailbox` instance, the provided message representation's contents are copied. In neither case is a reference to the message representation kept by - the :class:`Mailbox` instance. + the :class:`!Mailbox` instance. - The default :class:`Mailbox` iterator iterates over message representations, - not keys as the default dictionary iterator does. Moreover, modification of a + The default :class:`!Mailbox` :term:`iterator` iterates over message + representations, not keys as the default :class:`dictionary ` + iterator does. Moreover, modification of a mailbox during iteration is safe and well-defined. Messages added to the mailbox after an iterator is created will not be seen by the iterator. Messages removed from the mailbox before the iterator yields them @@ -69,14 +70,15 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Be very cautious when modifying mailboxes that might be simultaneously changed by some other process. The safest mailbox format to use for such - tasks is Maildir; try to avoid using single-file formats such as mbox for + tasks is :class:`Maildir`; try to avoid using single-file formats such as + :class:`mbox` for concurrent writing. If you're modifying a mailbox, you *must* lock it by calling the :meth:`lock` and :meth:`unlock` methods *before* reading any messages in the file or making any changes by adding or deleting a message. Failing to lock the mailbox runs the risk of losing messages or corrupting the entire mailbox. - :class:`Mailbox` instances have the following methods: + :class:`!Mailbox` instances have the following methods: .. method:: add(message) @@ -127,21 +129,23 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: iterkeys() - keys() - Return an iterator over all keys if called as :meth:`iterkeys` or return a - list of keys if called as :meth:`keys`. + Return an :term:`iterator` over all keys + + + .. method:: keys() + + The same as :meth:`iterkeys`, except that a :class:`list` is returned + rather than an :term:`iterator` .. method:: itervalues() __iter__() - values() - Return an iterator over representations of all messages if called as - :meth:`itervalues` or :meth:`__iter__` or return a list of such - representations if called as :meth:`values`. The messages are represented + Return an :term:`iterator` over representations of all messages. + The messages are represented as instances of the appropriate format-specific :class:`Message` subclass - unless a custom message factory was specified when the :class:`Mailbox` + unless a custom message factory was specified when the :class:`!Mailbox` instance was initialized. .. note:: @@ -150,15 +154,25 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. iterate over keys. + .. method:: values() + + The same as :meth:`itervalues`, except that a :class:`list` is returned + rather than an :term:`iterator` + + .. method:: iteritems() - items() - Return an iterator over (*key*, *message*) pairs, where *key* is a key and - *message* is a message representation, if called as :meth:`iteritems` or - return a list of such pairs if called as :meth:`items`. The messages are + Return an :term:`iterator` over (*key*, *message*) pairs, where *key* is + a key and *message* is a message representation. The messages are represented as instances of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. + when the :class:`!Mailbox` instance was initialized. + + + .. method:: items() + + The same as :meth:`iteritems`, except that a :class:`list` of pairs is + returned rather than an :term:`iterator` of pairs. .. method:: get(key, default=None) @@ -167,9 +181,9 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Return a representation of the message corresponding to *key*. If no such message exists, *default* is returned if the method was called as :meth:`get` and a :exc:`KeyError` exception is raised if the method was - called as :meth:`~object.__getitem__`. The message is represented as an instance + called as :meth:`!__getitem__`. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a - custom message factory was specified when the :class:`Mailbox` instance + custom message factory was specified when the :class:`!Mailbox` instance was initialized. @@ -198,21 +212,23 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: get_file(key) - Return a file-like representation of the message corresponding to *key*, + Return a :term:`file-like ` representation of the + message corresponding to *key*, or raise a :exc:`KeyError` exception if no such message exists. The file-like object behaves as if open in binary mode. This file should be closed once it is no longer needed. .. versionchanged:: 3.2 - The file object really is a binary file; previously it was incorrectly - returned in text mode. Also, the file-like object now supports the - context management protocol: you can use a :keyword:`with` statement to - automatically close it. + The file object really is a :term:`binary file`; previously it was + incorrectly returned in text mode. Also, the :term:`file-like object` + now supports the :term:`context manager` protocol: you can use a + :keyword:`with` statement to automatically close it. .. note:: - Unlike other representations of messages, file-like representations are - not necessarily independent of the :class:`Mailbox` instance that + Unlike other representations of messages, + :term:`file-like ` representations are not + necessarily independent of the :class:`!Mailbox` instance that created them or of the underlying mailbox. More specific documentation is provided by each subclass. @@ -238,7 +254,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. the message. If no such message exists, return *default*. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. + when the :class:`!Mailbox` instance was initialized. .. method:: popitem() @@ -248,7 +264,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. message. If the mailbox is empty, raise a :exc:`KeyError` exception. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. + when the :class:`!Mailbox` instance was initialized. .. method:: update(arg) @@ -259,7 +275,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. *message* as if by using :meth:`__setitem__`. As with :meth:`__setitem__`, each *key* must already correspond to a message in the mailbox or else a :exc:`KeyError` exception will be raised, so in general it is incorrect - for *arg* to be a :class:`Mailbox` instance. + for *arg* to be a :class:`!Mailbox` instance. .. note:: @@ -269,7 +285,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: flush() Write any pending changes to the filesystem. For some :class:`Mailbox` - subclasses, changes are always written immediately and :meth:`flush` does + subclasses, changes are always written immediately and :meth:`!flush` does nothing, but you should still make a habit of calling this method. @@ -290,13 +306,13 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: close() Flush the mailbox, unlock it if necessary, and close any open files. For - some :class:`Mailbox` subclasses, this method does nothing. + some :class:`!Mailbox` subclasses, this method does nothing. .. _mailbox-maildir: -:class:`Maildir` -^^^^^^^^^^^^^^^^ +:class:`!Maildir` objects +^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Maildir(dirname, factory=None, create=True) @@ -330,11 +346,11 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Folders of the style introduced by the Courier mail transfer agent are also supported. Any subdirectory of the main mailbox is considered a folder if ``'.'`` is the first character in its name. Folder names are represented by - :class:`Maildir` without the leading ``'.'``. Each folder is itself a Maildir + :class:`!Maildir` without the leading ``'.'``. Each folder is itself a Maildir mailbox but should not contain other folders. Instead, a logical nesting is indicated using ``'.'`` to delimit levels, e.g., "Archived.2005.07". - .. note:: + .. attribute:: Maildir.colon The Maildir specification requires the use of a colon (``':'``) in certain message file names. However, some operating systems do not permit this @@ -346,9 +362,12 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. import mailbox mailbox.Maildir.colon = '!' - The :attr:`colon` attribute may also be set on a per-instance basis. + The :attr:`!colon` attribute may also be set on a per-instance basis. - :class:`Maildir` instances have all of the methods of :class:`Mailbox` in + .. versionchanged:: 3.13 + :class:`Maildir` now ignores files with a leading dot. + + :class:`!Maildir` instances have all of the methods of :class:`Mailbox` in addition to the following: @@ -359,14 +378,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: get_folder(folder) - Return a :class:`Maildir` instance representing the folder whose name is + Return a :class:`!Maildir` instance representing the folder whose name is *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder does not exist. .. method:: add_folder(folder) - Create a folder whose name is *folder* and return a :class:`Maildir` + Create a folder whose name is *folder* and return a :class:`!Maildir` instance representing it. @@ -383,46 +402,6 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. last 36 hours. The Maildir specification says that mail-reading programs should do this occasionally. - Some :class:`Mailbox` methods implemented by :class:`Maildir` deserve special - remarks: - - - .. method:: add(message) - __setitem__(key, message) - update(arg) - - .. warning:: - - These methods generate unique file names based upon the current process - ID. When using multiple threads, undetected name clashes may occur and - cause corruption of the mailbox unless threads are coordinated to avoid - using these methods to manipulate the same mailbox simultaneously. - - - .. method:: flush() - - All changes to Maildir mailboxes are immediately applied, so this method - does nothing. - - - .. method:: lock() - unlock() - - Maildir mailboxes do not support (or require) locking, so these methods do - nothing. - - - .. method:: close() - - :class:`Maildir` instances do not keep any open files and the underlying - mailboxes do not support locking, so this method does nothing. - - - .. method:: get_file(key) - - Depending upon the host platform, it may not be possible to modify or - remove the underlying message while the returned file remains open. - .. method:: get_flags(key) @@ -525,6 +504,46 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. versionadded:: 3.13 + Some :class:`Mailbox` methods implemented by :class:`!Maildir` deserve special + remarks: + + + .. method:: add(message) + __setitem__(key, message) + update(arg) + + .. warning:: + + These methods generate unique file names based upon the current process + ID. When using multiple threads, undetected name clashes may occur and + cause corruption of the mailbox unless threads are coordinated to avoid + using these methods to manipulate the same mailbox simultaneously. + + + .. method:: flush() + + All changes to Maildir mailboxes are immediately applied, so this method + does nothing. + + + .. method:: lock() + unlock() + + Maildir mailboxes do not support (or require) locking, so these methods do + nothing. + + + .. method:: close() + + :class:`!Maildir` instances do not keep any open files and the underlying + mailboxes do not support locking, so this method does nothing. + + + .. method:: get_file(key) + + Depending upon the host platform, it may not be possible to modify or + remove the underlying message while the returned file remains open. + .. seealso:: @@ -539,8 +558,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-mbox: -:class:`mbox` -^^^^^^^^^^^^^ +:class:`!mbox` objects +^^^^^^^^^^^^^^^^^^^^^^ .. class:: mbox(path, factory=None, create=True) @@ -557,22 +576,22 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. each message indicated by a line whose first five characters are "From ". Several variations of the mbox format exist to address perceived shortcomings in - the original. In the interest of compatibility, :class:`mbox` implements the + the original. In the interest of compatibility, :class:`!mbox` implements the original format, which is sometimes referred to as :dfn:`mboxo`. This means that the :mailheader:`Content-Length` header, if present, is ignored and that any occurrences of "From " at the beginning of a line in a message body are transformed to ">From " when storing the message, although occurrences of ">From " are not transformed to "From " when reading the message. - Some :class:`Mailbox` methods implemented by :class:`mbox` deserve special + Some :class:`Mailbox` methods implemented by :class:`!mbox` deserve special remarks: .. method:: get_file(key) - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`mbox` instance may yield unpredictable results or raise an - exception. + Using the file after calling :meth:`~Mailbox.flush` or + :meth:`~Mailbox.close` on the :class:`!mbox` instance may yield + unpredictable results or raise an exception. .. method:: lock() @@ -596,8 +615,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-mh: -:class:`MH` -^^^^^^^^^^^ +:class:`!MH` objects +^^^^^^^^^^^^^^^^^^^^ .. class:: MH(path, factory=None, create=True) @@ -617,14 +636,18 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. messages without moving them to sub-folders. Sequences are defined in a file called :file:`.mh_sequences` in each folder. - The :class:`MH` class manipulates MH mailboxes, but it does not attempt to + The :class:`!MH` class manipulates MH mailboxes, but it does not attempt to emulate all of :program:`mh`'s behaviors. In particular, it does not modify and is not affected by the :file:`context` or :file:`.mh_profile` files that are used by :program:`mh` to store its state and configuration. - :class:`MH` instances have all of the methods of :class:`Mailbox` in addition + :class:`!MH` instances have all of the methods of :class:`Mailbox` in addition to the following: + .. versionchanged:: 3.13 + + Supported folders that don't contain a :file:`.mh_sequences` file. + .. method:: list_folders() @@ -633,14 +656,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: get_folder(folder) - Return an :class:`MH` instance representing the folder whose name is + Return an :class:`!MH` instance representing the folder whose name is *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder does not exist. .. method:: add_folder(folder) - Create a folder whose name is *folder* and return an :class:`MH` instance + Create a folder whose name is *folder* and return an :class:`!MH` instance representing it. @@ -674,7 +697,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Already-issued keys are invalidated by this operation and should not be subsequently used. - Some :class:`Mailbox` methods implemented by :class:`MH` deserve special + Some :class:`Mailbox` methods implemented by :class:`!MH` deserve special remarks: @@ -710,7 +733,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: close() - :class:`MH` instances do not keep any open files, so this method is + :class:`!MH` instances do not keep any open files, so this method is equivalent to :meth:`unlock`. @@ -726,8 +749,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-babyl: -:class:`Babyl` -^^^^^^^^^^^^^^ +:class:`!Babyl` objects +^^^^^^^^^^^^^^^^^^^^^^^ .. class:: Babyl(path, factory=None, create=True) @@ -754,7 +777,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. message, and a list of all user-defined labels found in the mailbox is kept in the Babyl options section. - :class:`Babyl` instances have all of the methods of :class:`Mailbox` in + :class:`!Babyl` instances have all of the methods of :class:`Mailbox` in addition to the following: @@ -769,7 +792,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. options section, but the Babyl section is updated whenever the mailbox is modified. - Some :class:`Mailbox` methods implemented by :class:`Babyl` deserve special + Some :class:`Mailbox` methods implemented by :class:`!Babyl` deserve special remarks: @@ -802,8 +825,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-mmdf: -:class:`MMDF` -^^^^^^^^^^^^^ +:class:`!MMDF` objects +^^^^^^^^^^^^^^^^^^^^^^ .. class:: MMDF(path, factory=None, create=True) @@ -824,15 +847,15 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. ">From " when storing messages because the extra message separator lines prevent mistaking such occurrences for the starts of subsequent messages. - Some :class:`Mailbox` methods implemented by :class:`MMDF` deserve special + Some :class:`Mailbox` methods implemented by :class:`!MMDF` deserve special remarks: .. method:: get_file(key) - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`MMDF` instance may yield unpredictable results or raise an - exception. + Using the file after calling :meth:`~Mailbox.flush` or + :meth:`~Mailbox.close` on the :class:`!MMDF` instance may yield + unpredictable results or raise an exception. .. method:: lock() @@ -854,20 +877,20 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. _mailbox-message-objects: -:class:`Message` objects ------------------------- +:class:`!Message` objects +------------------------- .. class:: Message(message=None) A subclass of the :mod:`email.message` module's - :class:`~email.message.Message`. Subclasses of :class:`mailbox.Message` add + :class:`~email.message.Message`. Subclasses of :class:`!mailbox.Message` add mailbox-format-specific state and behavior. If *message* is omitted, the new instance is created in a default, empty state. If *message* is an :class:`email.message.Message` instance, its contents are copied; furthermore, any format-specific information is converted insofar as - possible if *message* is a :class:`Message` instance. If *message* is a string, + possible if *message* is a :class:`!Message` instance. If *message* is a string, a byte string, or a file, it should contain an :rfc:`2822`\ -compliant message, which is read and parsed. Files should be open in binary mode, but text mode files @@ -882,18 +905,18 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. such as whether a message has been read by the user or marked as important is retained, because it applies to the message itself. - There is no requirement that :class:`Message` instances be used to represent + There is no requirement that :class:`!Message` instances be used to represent messages retrieved using :class:`Mailbox` instances. In some situations, the - time and memory required to generate :class:`Message` representations might - not be acceptable. For such situations, :class:`Mailbox` instances also + time and memory required to generate :class:`!Message` representations might + not be acceptable. For such situations, :class:`!Mailbox` instances also offer string and file-like representations, and a custom message factory may - be specified when a :class:`Mailbox` instance is initialized. + be specified when a :class:`!Mailbox` instance is initialized. .. _mailbox-maildirmessage: -:class:`MaildirMessage` -^^^^^^^^^^^^^^^^^^^^^^^ +:class:`!MaildirMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: MaildirMessage(message=None) @@ -928,7 +951,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. | T | Trashed | Marked for subsequent deletion | +------+---------+--------------------------------+ - :class:`MaildirMessage` instances offer the following methods: + :class:`!MaildirMessage` instances offer the following methods: .. method:: get_subdir() @@ -1005,7 +1028,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Set "info" to *info*, which should be a string. -When a :class:`MaildirMessage` instance is created based upon an +When a :class:`!MaildirMessage` instance is created based upon an :class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` and :mailheader:`X-Status` headers are omitted and the following conversions take place: @@ -1025,7 +1048,7 @@ take place: | T flag | D flag | +--------------------+----------------------------------------------+ -When a :class:`MaildirMessage` instance is created based upon an +When a :class:`!MaildirMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +-------------------------------+--------------------------+ @@ -1040,7 +1063,7 @@ When a :class:`MaildirMessage` instance is created based upon an | R flag | "replied" sequence | +-------------------------------+--------------------------+ -When a :class:`MaildirMessage` instance is created based upon a +When a :class:`!MaildirMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +-------------------------------+-------------------------------+ @@ -1060,8 +1083,8 @@ When a :class:`MaildirMessage` instance is created based upon a .. _mailbox-mboxmessage: -:class:`mboxMessage` -^^^^^^^^^^^^^^^^^^^^ +:class:`!mboxMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: mboxMessage(message=None) @@ -1097,7 +1120,7 @@ When a :class:`MaildirMessage` instance is created based upon a "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The flags and headers typically appear in the order mentioned. - :class:`mboxMessage` instances offer the following methods: + :class:`!mboxMessage` instances offer the following methods: .. method:: get_from() @@ -1145,7 +1168,7 @@ When a :class:`MaildirMessage` instance is created based upon a remove more than one flag at a time, *flag* maybe a string of more than one character. -When an :class:`mboxMessage` instance is created based upon a +When an :class:`!mboxMessage` instance is created based upon a :class:`MaildirMessage` instance, a "From " line is generated based upon the :class:`MaildirMessage` instance's delivery date, and the following conversions take place: @@ -1164,7 +1187,7 @@ take place: | A flag | R flag | +-----------------+-------------------------------+ -When an :class:`mboxMessage` instance is created based upon an +When an :class:`!mboxMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +-------------------+--------------------------+ @@ -1179,7 +1202,7 @@ When an :class:`mboxMessage` instance is created based upon an | A flag | "replied" sequence | +-------------------+--------------------------+ -When an :class:`mboxMessage` instance is created based upon a +When an :class:`!mboxMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +-------------------+-----------------------------+ @@ -1194,7 +1217,8 @@ When an :class:`mboxMessage` instance is created based upon a | A flag | "answered" label | +-------------------+-----------------------------+ -When a :class:`Message` instance is created based upon an :class:`MMDFMessage` +When a :class:`!mboxMessage` instance is created based upon an +:class:`MMDFMessage` instance, the "From " line is copied and all flags directly correspond: +-----------------+----------------------------+ @@ -1214,8 +1238,8 @@ instance, the "From " line is copied and all flags directly correspond: .. _mailbox-mhmessage: -:class:`MHMessage` -^^^^^^^^^^^^^^^^^^ +:class:`!MHMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: MHMessage(message=None) @@ -1239,7 +1263,7 @@ instance, the "From " line is copied and all flags directly correspond: | flagged | Marked as important | +----------+------------------------------------------+ - :class:`MHMessage` instances offer the following methods: + :class:`!MHMessage` instances offer the following methods: .. method:: get_sequences() @@ -1261,7 +1285,7 @@ instance, the "From " line is copied and all flags directly correspond: Remove *sequence* from the list of sequences that include this message. -When an :class:`MHMessage` instance is created based upon a +When an :class:`!MHMessage` instance is created based upon a :class:`MaildirMessage` instance, the following conversions take place: +--------------------+-------------------------------+ @@ -1274,7 +1298,7 @@ When an :class:`MHMessage` instance is created based upon a | "flagged" sequence | F flag | +--------------------+-------------------------------+ -When an :class:`MHMessage` instance is created based upon an +When an :class:`!MHMessage` instance is created based upon an :class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` and :mailheader:`X-Status` headers are omitted and the following conversions take place: @@ -1290,7 +1314,7 @@ take place: | "flagged" sequence | F flag | +--------------------+----------------------------------------------+ -When an :class:`MHMessage` instance is created based upon a +When an :class:`!MHMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +--------------------+-----------------------------+ @@ -1304,8 +1328,8 @@ When an :class:`MHMessage` instance is created based upon a .. _mailbox-babylmessage: -:class:`BabylMessage` -^^^^^^^^^^^^^^^^^^^^^ +:class:`!BabylMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: BabylMessage(message=None) @@ -1334,11 +1358,11 @@ When an :class:`MHMessage` instance is created based upon a | resent | Resent | +-----------+------------------------------------------+ - By default, Rmail displays only visible headers. The :class:`BabylMessage` + By default, Rmail displays only visible headers. The :class:`!BabylMessage` class, though, uses the original headers because they are more complete. Visible headers may be accessed explicitly if desired. - :class:`BabylMessage` instances offer the following methods: + :class:`!BabylMessage` instances offer the following methods: .. method:: get_labels() @@ -1377,7 +1401,7 @@ When an :class:`MHMessage` instance is created based upon a .. method:: update_visible() - When a :class:`BabylMessage` instance's original headers are modified, the + When a :class:`!BabylMessage` instance's original headers are modified, the visible headers are not automatically modified to correspond. This method updates the visible headers as follows: each visible header with a corresponding original header is set to the value of the original header, @@ -1387,7 +1411,7 @@ When an :class:`MHMessage` instance is created based upon a present in the original headers but not the visible headers are added to the visible headers. -When a :class:`BabylMessage` instance is created based upon a +When a :class:`!BabylMessage` instance is created based upon a :class:`MaildirMessage` instance, the following conversions take place: +-------------------+-------------------------------+ @@ -1402,7 +1426,7 @@ When a :class:`BabylMessage` instance is created based upon a | "forwarded" label | P flag | +-------------------+-------------------------------+ -When a :class:`BabylMessage` instance is created based upon an +When a :class:`!BabylMessage` instance is created based upon an :class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` and :mailheader:`X-Status` headers are omitted and the following conversions take place: @@ -1418,7 +1442,7 @@ take place: | "answered" label | A flag | +------------------+----------------------------------------------+ -When a :class:`BabylMessage` instance is created based upon an +When a :class:`!BabylMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +------------------+--------------------------+ @@ -1432,8 +1456,8 @@ When a :class:`BabylMessage` instance is created based upon an .. _mailbox-mmdfmessage: -:class:`MMDFMessage` -^^^^^^^^^^^^^^^^^^^^ +:class:`!MMDFMessage` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: MMDFMessage(message=None) @@ -1467,7 +1491,7 @@ When a :class:`BabylMessage` instance is created based upon an "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The flags and headers typically appear in the order mentioned. - :class:`MMDFMessage` instances offer the following methods, which are + :class:`!MMDFMessage` instances offer the following methods, which are identical to those offered by :class:`mboxMessage`: @@ -1516,7 +1540,7 @@ When a :class:`BabylMessage` instance is created based upon an remove more than one flag at a time, *flag* maybe a string of more than one character. -When an :class:`MMDFMessage` instance is created based upon a +When an :class:`!MMDFMessage` instance is created based upon a :class:`MaildirMessage` instance, a "From " line is generated based upon the :class:`MaildirMessage` instance's delivery date, and the following conversions take place: @@ -1535,7 +1559,7 @@ take place: | A flag | R flag | +-----------------+-------------------------------+ -When an :class:`MMDFMessage` instance is created based upon an +When an :class:`!MMDFMessage` instance is created based upon an :class:`MHMessage` instance, the following conversions take place: +-------------------+--------------------------+ @@ -1550,7 +1574,7 @@ When an :class:`MMDFMessage` instance is created based upon an | A flag | "replied" sequence | +-------------------+--------------------------+ -When an :class:`MMDFMessage` instance is created based upon a +When an :class:`!MMDFMessage` instance is created based upon a :class:`BabylMessage` instance, the following conversions take place: +-------------------+-----------------------------+ @@ -1565,7 +1589,7 @@ When an :class:`MMDFMessage` instance is created based upon a | A flag | "answered" label | +-------------------+-----------------------------+ -When an :class:`MMDFMessage` instance is created based upon an +When an :class:`!MMDFMessage` instance is created based upon an :class:`mboxMessage` instance, the "From " line is copied and all flags directly correspond: @@ -1587,7 +1611,7 @@ correspond: Exceptions ---------- -The following exception classes are defined in the :mod:`mailbox` module: +The following exception classes are defined in the :mod:`!mailbox` module: .. exception:: Error() diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index f453e6403d932d..671130d9b29fc0 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -36,7 +36,7 @@ or other communications requiring the serialization/deserialization and copying of data. -.. class:: SharedMemory(name=None, create=False, size=0) +.. class:: SharedMemory(name=None, create=False, size=0, *, track=True) Creates a new shared memory block or attaches to an existing shared memory block. Each shared memory block is assigned a unique name. @@ -64,26 +64,45 @@ copying of data. memory block may be larger or equal to the size requested. When attaching to an existing shared memory block, the ``size`` parameter is ignored. + *track*, when enabled, registers the shared memory block with a resource + tracker process on platforms where the OS does not do this automatically. + The resource tracker ensures proper cleanup of the shared memory even + if all other processes with access to the memory exit without doing so. + Python processes created from a common ancestor using :mod:`multiprocessing` + facilities share a single resource tracker process, and the lifetime of + shared memory segments is handled automatically among these processes. + Python processes created in any other way will receive their own + resource tracker when accessing shared memory with *track* enabled. + This will cause the shared memory to be deleted by the resource tracker + of the first process that terminates. + To avoid this issue, users of :mod:`subprocess` or standalone Python + processes should set *track* to ``False`` when there is already another + process in place that does the bookkeeping. + *track* is ignored on Windows, which has its own tracking and + automatically deletes shared memory when all handles to it have been closed. + + .. versionchanged:: 3.13 Added *track* parameter. + .. method:: close() - Closes access to the shared memory from this instance. In order to - ensure proper cleanup of resources, all instances should call - ``close()`` once the instance is no longer needed. Note that calling - ``close()`` does not cause the shared memory block itself to be - destroyed. + Closes the file descriptor/handle to the shared memory from this + instance. :meth:`close()` should be called once access to the shared + memory block from this instance is no longer needed. Depending + on operating system, the underlying memory may or may not be freed + even if all handles to it have been closed. To ensure proper cleanup, + use the :meth:`unlink()` method. .. method:: unlink() - Requests that the underlying shared memory block be destroyed. In - order to ensure proper cleanup of resources, ``unlink()`` should be - called once (and only once) across all processes which have need - for the shared memory block. After requesting its destruction, a - shared memory block may or may not be immediately destroyed and - this behavior may differ across platforms. Attempts to access data - inside the shared memory block after ``unlink()`` has been called may - result in memory access errors. Note: the last process relinquishing - its hold on a shared memory block may call ``unlink()`` and - :meth:`close()` in either order. + Deletes the underlying shared memory block. This should be called only + once per shared memory block regardless of the number of handles to it, + even in other processes. + :meth:`unlink()` and :meth:`close()` can be called in any order, but + trying to access data inside a shared memory block after :meth:`unlink()` + may result in memory access errors, depending on platform. + + This method has no effect on Windows, where the only way to delete a + shared memory block is to close all handles. .. attribute:: buf diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 2a05b56db051f9..17d1a275f04c9b 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`numbers` module (:pep:`3141`) defines a hierarchy of numeric +The :mod:`!numbers` module (:pep:`3141`) defines a hierarchy of numeric :term:`abstract base classes ` which progressively define more operations. None of the types defined in this module are intended to be instantiated. @@ -45,7 +45,7 @@ The numeric tower .. class:: Real - To :class:`Complex`, :class:`Real` adds the operations that work on real + To :class:`Complex`, :class:`!Real` adds the operations that work on real numbers. In short, those are: a conversion to :class:`float`, :func:`math.trunc`, @@ -126,7 +126,8 @@ We want to implement the arithmetic operations so that mixed-mode operations either call an implementation whose author knew about the types of both arguments, or convert both to the nearest built in type and do the operation there. For subtypes of :class:`Integral`, this -means that :meth:`__add__` and :meth:`__radd__` should be defined as:: +means that :meth:`~object.__add__` and :meth:`~object.__radd__` should be +defined as:: class MyIntegral(Integral): @@ -160,15 +161,15 @@ refer to ``MyIntegral`` and ``OtherTypeIKnowAbout`` as of :class:`Complex` (``a : A <: Complex``), and ``b : B <: Complex``. I'll consider ``a + b``: -1. If ``A`` defines an :meth:`__add__` which accepts ``b``, all is +1. If ``A`` defines an :meth:`~object.__add__` which accepts ``b``, all is well. 2. If ``A`` falls back to the boilerplate code, and it were to - return a value from :meth:`__add__`, we'd miss the possibility - that ``B`` defines a more intelligent :meth:`__radd__`, so the + return a value from :meth:`~object.__add__`, we'd miss the possibility + that ``B`` defines a more intelligent :meth:`~object.__radd__`, so the boilerplate should return :const:`NotImplemented` from - :meth:`__add__`. (Or ``A`` may not implement :meth:`__add__` at + :meth:`!__add__`. (Or ``A`` may not implement :meth:`!__add__` at all.) -3. Then ``B``'s :meth:`__radd__` gets a chance. If it accepts +3. Then ``B``'s :meth:`~object.__radd__` gets a chance. If it accepts ``a``, all is well. 4. If it falls back to the boilerplate, there are no more possible methods to try, so this is where the default implementation @@ -180,7 +181,7 @@ Complex``. I'll consider ``a + b``: If ``A <: Complex`` and ``B <: Real`` without sharing any other knowledge, then the appropriate shared operation is the one involving the built -in :class:`complex`, and both :meth:`__radd__` s land there, so ``a+b +in :class:`complex`, and both :meth:`~object.__radd__` s land there, so ``a+b == b+a``. Because most of the operations on any given type will be very similar, diff --git a/Doc/library/os.rst b/Doc/library/os.rst index fe573f188ab066..637191f2980a05 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1001,11 +1001,14 @@ as internal buffering of data. .. audit-event:: os.chmod path,mode,dir_fd os.fchmod - .. availability:: Unix. + .. availability:: Unix, Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. + .. versionchanged:: 3.13 + Added support on Windows. + .. function:: fchown(fd, uid, gid) @@ -2062,6 +2065,7 @@ features: Although Windows supports :func:`chmod`, you can only set the file's read-only flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` constants or a corresponding integer value). All other bits are ignored. + The default value of *follow_symlinks* is ``False`` on Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. @@ -2075,6 +2079,10 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Added support for a file descriptor and the *follow_symlinks* argument + on Windows. + .. function:: chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True) @@ -2160,13 +2168,19 @@ features: for possible values of *mode*. As of Python 3.3, this is equivalent to ``os.chmod(path, mode, follow_symlinks=False)``. + ``lchmod()`` is not part of POSIX, but Unix implementations may have it if + changing the mode of symbolic links is supported. + .. audit-event:: os.chmod path,mode,dir_fd os.lchmod - .. availability:: Unix. + .. availability:: Unix, Windows, not Linux, FreeBSD >= 1.3, NetBSD >= 1.3, not OpenBSD .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Added support on Windows. + .. function:: lchown(path, uid, gid) Change the owner and group id of *path* to the numeric *uid* and *gid*. This @@ -4170,7 +4184,7 @@ to be ignored. The "l" and "v" variants of the :func:`exec\* ` functions differ in how command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the - individual parameters simply become additional parameters to the :func:`execl\*` + individual parameters simply become additional parameters to the :func:`!execl\*` functions. The "v" variants are good when the number of parameters is variable, with the arguments being passed in a list or tuple as the *args* parameter. In either case, the arguments to the child process should start with @@ -4373,6 +4387,11 @@ written in Python, such as a mail server's external command delivery program. If you use TLS sockets in an application calling ``fork()``, see the warning in the :mod:`ssl` documentation. + .. warning:: + + On macOS the use of this function is unsafe when mixed with using + higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.8 Calling ``fork()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). @@ -4412,6 +4431,11 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.forkpty "" os.forkpty + .. warning:: + + On macOS the use of this function is unsafe when mixed with using + higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.12 If Python is able to detect that your process has multiple threads, this now raises a :exc:`DeprecationWarning`. See the @@ -4550,7 +4574,8 @@ written in Python, such as a mail server's external command delivery program. Most users should use :func:`subprocess.run` instead of :func:`posix_spawn`. The positional-only arguments *path*, *args*, and *env* are similar to - :func:`execve`. + :func:`execve`. *env* is allowed to be ``None``, in which case current + process' environment is used. The *path* parameter is the path to the executable file. The *path* should contain a directory. Use :func:`posix_spawnp` to pass an executable file @@ -4580,10 +4605,17 @@ written in Python, such as a mail server's external command delivery program. Performs ``os.dup2(fd, new_fd)``. + .. data:: POSIX_SPAWN_CLOSEFROM + + (``os.POSIX_SPAWN_CLOSEFROM``, *fd*) + + Performs ``os.closerange(fd, INF)``. + These tuples correspond to the C library :c:func:`!posix_spawn_file_actions_addopen`, - :c:func:`!posix_spawn_file_actions_addclose`, and - :c:func:`!posix_spawn_file_actions_adddup2` API calls used to prepare + :c:func:`!posix_spawn_file_actions_addclose`, + :c:func:`!posix_spawn_file_actions_adddup2`, and + :c:func:`!posix_spawn_file_actions_addclosefrom_np` API calls used to prepare for the :c:func:`!posix_spawn` call itself. The *setpgroup* argument will set the process group of the child to the value @@ -4625,6 +4657,13 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.8 + .. versionchanged:: 3.13 + *env* parameter accepts ``None``. + + .. versionchanged:: 3.13 + ``os.POSIX_SPAWN_CLOSEFROM`` is available on platforms where + :c:func:`!posix_spawn_file_actions_addclosefrom_np` exists. + .. availability:: Unix, not Emscripten, not WASI. .. function:: posix_spawnp(path, argv, env, *, file_actions=None, \ @@ -4708,7 +4747,7 @@ written in Python, such as a mail server's external command delivery program. command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the individual parameters simply become additional parameters to the - :func:`spawnl\*` functions. The "v" variants are good when the number of + :func:`!spawnl\*` functions. The "v" variants are good when the number of parameters is variable, with the arguments being passed in a list or tuple as the *args* parameter. In either case, the arguments to the child process must start with the name of the command being run. @@ -4758,7 +4797,7 @@ written in Python, such as a mail server's external command delivery program. P_NOWAITO Possible values for the *mode* parameter to the :func:`spawn\* ` family of - functions. If either of these values is given, the :func:`spawn\*` functions + functions. If either of these values is given, the :func:`spawn\* ` functions will return as soon as the new process has been created, with the process id as the return value. @@ -4768,7 +4807,7 @@ written in Python, such as a mail server's external command delivery program. .. data:: P_WAIT Possible value for the *mode* parameter to the :func:`spawn\* ` family of - functions. If this is given as *mode*, the :func:`spawn\*` functions will not + functions. If this is given as *mode*, the :func:`spawn\* ` functions will not return until the new process has run to completion and will return the exit code of the process the run is successful, or ``-signal`` if a signal kills the process. @@ -4949,6 +4988,9 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.3 + .. versionchanged:: 3.13 + This function is now available on macOS as well. + .. function:: waitpid(pid, options, /) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 7ecfd120db8d15..60791725c2323d 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -306,7 +306,7 @@ Pure paths provide the following methods and properties: .. attribute:: PurePath.pathmod The implementation of the :mod:`os.path` module used for low-level path - operations: either ``posixpath`` or ``ntpath``. + operations: either :mod:`posixpath` or :mod:`ntpath`. .. versionadded:: 3.13 @@ -595,6 +595,9 @@ Pure paths provide the following methods and properties: >>> PurePath('a/b.py').match(pattern) True + .. versionchanged:: 3.12 + Accepts an object implementing the :class:`os.PathLike` interface. + As with other methods, case-sensitivity follows platform defaults:: >>> PurePosixPath('b.py').match('*.PY') @@ -1017,15 +1020,21 @@ call fails (for example because the path doesn't exist). future Python release, patterns with this ending will match both files and directories. Add a trailing slash to match only directories. -.. method:: Path.group() +.. method:: Path.group(*, follow_symlinks=True) - Return the name of the group owning the file. :exc:`KeyError` is raised + Return the name of the group owning the file. :exc:`KeyError` is raised if the file's gid isn't found in the system database. + This method normally follows symlinks; to get the group of the symlink, add + the argument ``follow_symlinks=False``. + .. versionchanged:: 3.13 Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not available. In previous versions, :exc:`NotImplementedError` was raised. + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + .. method:: Path.is_dir(*, follow_symlinks=True) @@ -1291,15 +1300,21 @@ call fails (for example because the path doesn't exist). '#!/usr/bin/env python3\n' -.. method:: Path.owner() +.. method:: Path.owner(*, follow_symlinks=True) - Return the name of the user owning the file. :exc:`KeyError` is raised + Return the name of the user owning the file. :exc:`KeyError` is raised if the file's uid isn't found in the system database. + This method normally follows symlinks; to get the owner of the symlink, add + the argument ``follow_symlinks=False``. + .. versionchanged:: 3.13 Raises :exc:`UnsupportedOperation` if the :mod:`pwd` module is not available. In previous versions, :exc:`NotImplementedError` was raised. + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + .. method:: Path.read_bytes() diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index bbc6aacc62aafa..2495dcf50bb17f 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -570,10 +570,27 @@ can be overridden by the local file. Start an interactive interpreter (using the :mod:`code` module) whose global namespace contains all the (global and local) names found in the current - scope. + scope. Use ``exit()`` or ``quit()`` to exit the interpreter and return to + the debugger. + + .. note:: + + Because interact creates a new global namespace with the current global + and local namespace for execution, assignment to variables will not + affect the original namespaces. + However, modification to the mutable objects will be reflected in the + original namespaces. .. versionadded:: 3.2 + .. versionadded:: 3.13 + ``exit()`` and ``quit()`` can be used to exit :pdbcmd:`interact` + command. + + .. versionchanged:: 3.13 + :pdbcmd:`interact` directs its output to the debugger's + output channel rather than :data:`sys.stderr`. + .. _debugger-aliases: .. pdbcommand:: alias [name [command]] diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 732ef3536863cc..10f1a48fc70a72 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -52,7 +52,7 @@ or :class:`datetime.datetime` objects. This module defines the following functions: -.. function:: load(fp, *, fmt=None, dict_type=dict) +.. function:: load(fp, *, fmt=None, dict_type=dict, aware_datetime=False) Read a plist file. *fp* should be a readable and binary file object. Return the unpacked root object (which usually is a @@ -69,6 +69,10 @@ This module defines the following functions: The *dict_type* is the type used for dictionaries that are read from the plist file. + When *aware_datetime* is true, fields with type ``datetime.datetime`` will + be created as :ref:`aware object `, with + :attr:`!tzinfo` as :attr:`datetime.UTC`. + XML data for the :data:`FMT_XML` format is parsed using the Expat parser from :mod:`xml.parsers.expat` -- see its documentation for possible exceptions on ill-formed XML. Unknown elements will simply be ignored @@ -79,8 +83,11 @@ This module defines the following functions: .. versionadded:: 3.4 + .. versionchanged:: 3.13 + The keyword-only parameter *aware_datetime* has been added. + -.. function:: loads(data, *, fmt=None, dict_type=dict) +.. function:: loads(data, *, fmt=None, dict_type=dict, aware_datetime=False) Load a plist from a bytes object. See :func:`load` for an explanation of the keyword arguments. @@ -88,7 +95,7 @@ This module defines the following functions: .. versionadded:: 3.4 -.. function:: dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, aware_datetime=False) Write *value* to a plist file. *Fp* should be a writable, binary file object. @@ -107,6 +114,10 @@ This module defines the following functions: When *skipkeys* is false (the default) the function raises :exc:`TypeError` when a key of a dictionary is not a string, otherwise such keys are skipped. + When *aware_datetime* is true and any field with type ``datetime.datetime`` + is set as a :ref:`aware object `, it will convert to + UTC timezone before writing it. + A :exc:`TypeError` will be raised if the object is of an unsupported type or a container that contains objects of unsupported types. @@ -115,8 +126,11 @@ This module defines the following functions: .. versionadded:: 3.4 + .. versionchanged:: 3.13 + The keyword-only parameter *aware_datetime* has been added. + -.. function:: dumps(value, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dumps(value, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, aware_datetime=False) Return *value* as a plist-formatted bytes object. See the documentation for :func:`dump` for an explanation of the keyword diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index af9378464edb9f..bd2f5ed45cb8b4 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -33,6 +33,9 @@ The :mod:`pty` module defines the following functions: file descriptor connected to the child's controlling terminal (and also to the child's standard input and output). + .. warning:: On macOS the use of this function is unsafe when mixed with using + higher-level system APIs, and that includes using :mod:`urllib.request`. + .. function:: openpty() diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 76ae97a8be7e63..d0ced2416c9578 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -34,10 +34,8 @@ instance of the :class:`random.Random` class. You can instantiate your own instances of :class:`Random` to get generators that don't share state. Class :class:`Random` can also be subclassed if you want to use a different -basic generator of your own devising: in that case, override the :meth:`~Random.random`, -:meth:`~Random.seed`, :meth:`~Random.getstate`, and :meth:`~Random.setstate` methods. -Optionally, a new generator can supply a :meth:`~Random.getrandbits` method --- this -allows :meth:`randrange` to produce selections over an arbitrarily large range. +basic generator of your own devising: see the documentation on that class for +more details. The :mod:`random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers @@ -88,7 +86,7 @@ Bookkeeping functions .. versionchanged:: 3.11 The *seed* must be one of the following types: - *NoneType*, :class:`int`, :class:`float`, :class:`str`, + ``None``, :class:`int`, :class:`float`, :class:`str`, :class:`bytes`, or :class:`bytearray`. .. function:: getstate() @@ -220,8 +218,8 @@ Functions for sequences generated. For example, a sequence of length 2080 is the largest that can fit within the period of the Mersenne Twister random number generator. - .. deprecated-removed:: 3.9 3.11 - The optional parameter *random*. + .. versionchanged:: 3.11 + Removed the optional parameter *random*. .. function:: sample(population, k, *, counts=None) @@ -407,11 +405,42 @@ Alternative Generator Class that implements the default pseudo-random number generator used by the :mod:`random` module. - .. deprecated-removed:: 3.9 3.11 + .. versionchanged:: 3.11 Formerly the *seed* could be any hashable object. Now it is limited to: - :class:`NoneType`, :class:`int`, :class:`float`, :class:`str`, + ``None``, :class:`int`, :class:`float`, :class:`str`, :class:`bytes`, or :class:`bytearray`. + Subclasses of :class:`!Random` should override the following methods if they + wish to make use of a different basic generator: + + .. method:: Random.seed(a=None, version=2) + + Override this method in subclasses to customise the :meth:`~random.seed` + behaviour of :class:`!Random` instances. + + .. method:: Random.getstate() + + Override this method in subclasses to customise the :meth:`~random.getstate` + behaviour of :class:`!Random` instances. + + .. method:: Random.setstate(state) + + Override this method in subclasses to customise the :meth:`~random.setstate` + behaviour of :class:`!Random` instances. + + .. method:: Random.random() + + Override this method in subclasses to customise the :meth:`~random.random` + behaviour of :class:`!Random` instances. + + Optionally, a custom generator subclass can also supply the following method: + + .. method:: Random.getrandbits(k) + + Override this method in subclasses to customise the + :meth:`~random.getrandbits` behaviour of :class:`!Random` instances. + + .. class:: SystemRandom([seed]) Class that uses the :func:`os.urandom` function for generating random numbers @@ -445,30 +474,30 @@ Examples Basic examples:: - >>> random() # Random float: 0.0 <= x < 1.0 + >>> random() # Random float: 0.0 <= x < 1.0 0.37444887175646646 - >>> uniform(2.5, 10.0) # Random float: 2.5 <= x <= 10.0 + >>> uniform(2.5, 10.0) # Random float: 2.5 <= x <= 10.0 3.1800146073117523 - >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds + >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds 5.148957571865031 - >>> randrange(10) # Integer from 0 to 9 inclusive + >>> randrange(10) # Integer from 0 to 9 inclusive 7 - >>> randrange(0, 101, 2) # Even integer from 0 to 100 inclusive + >>> randrange(0, 101, 2) # Even integer from 0 to 100 inclusive 26 - >>> choice(['win', 'lose', 'draw']) # Single random element from a sequence + >>> choice(['win', 'lose', 'draw']) # Single random element from a sequence 'draw' >>> deck = 'ace two three four'.split() - >>> shuffle(deck) # Shuffle a list + >>> shuffle(deck) # Shuffle a list >>> deck ['four', 'two', 'ace', 'three'] - >>> sample([10, 20, 30, 40, 50], k=4) # Four samples without replacement + >>> sample([10, 20, 30, 40, 50], k=4) # Four samples without replacement [40, 10, 50, 30] Simulations:: @@ -572,14 +601,14 @@ Simulation of arrival times and service deliveries for a multiserver queue:: including simulation, sampling, shuffling, and cross-validation. `Economics Simulation - `_ + `_ a simulation of a marketplace by `Peter Norvig `_ that shows effective use of many of the tools and distributions provided by this module (gauss, uniform, sample, betavariate, choice, triangular, and randrange). `A Concrete Introduction to Probability (using Python) - `_ + `_ a tutorial by `Peter Norvig `_ covering the basics of probability theory, how to write simulations, and how to perform data analysis using Python. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 251ec8ca0021a6..302f7224de4a7a 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1093,12 +1093,12 @@ Functions Exceptions ^^^^^^^^^^ -.. exception:: error(msg, pattern=None, pos=None) +.. exception:: PatternError(msg, pattern=None, pos=None) Exception raised when a string passed to one of the functions here is not a valid regular expression (for example, it might contain unmatched parentheses) or when some other error occurs during compilation or matching. It is never an - error if a string contains no match for a pattern. The error instance has + error if a string contains no match for a pattern. The ``PatternError`` instance has the following additional attributes: .. attribute:: msg @@ -1124,6 +1124,10 @@ Exceptions .. versionchanged:: 3.5 Added additional attributes. + .. versionchanged:: 3.13 + ``PatternError`` was originally named ``error``; the latter is kept as an alias for + backward compatibility. + .. _re-objects: Regular Expression Objects diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 8fb0eca8df74d8..1adafcaa02eab9 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -27,16 +27,15 @@ Readline library in general. .. note:: The underlying Readline library API may be implemented by - the ``libedit`` library instead of GNU readline. + the ``editline`` (``libedit``) library instead of GNU readline. On macOS the :mod:`readline` module detects which library is being used at run time. - The configuration file for ``libedit`` is different from that + The configuration file for ``editline`` is different from that of GNU readline. If you programmatically load configuration strings - you can check for the text "libedit" in :const:`readline.__doc__` - to differentiate between GNU readline and libedit. + you can use :data:`backend` to determine which library is being used. - If you use *editline*/``libedit`` readline emulation on macOS, the + If you use ``editline``/``libedit`` readline emulation on macOS, the initialization file located in your home directory is named ``.editrc``. For example, the following content in ``~/.editrc`` will turn ON *vi* keybindings and TAB completion:: @@ -44,6 +43,12 @@ Readline library in general. python:bind -v python:bind ^I rl_complete +.. data:: backend + + The name of the underlying Readline library being used, either + ``"readline"`` or ``"editline"``. + + .. versionadded:: 3.13 Init file --------- @@ -213,6 +218,8 @@ Startup hooks if Python was compiled for a version of the library that supports it. +.. _readline-completion: + Completion ---------- diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 5ebb0a7780c37b..678a11c6f45490 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`reprlib` module provides a means for producing object representations +The :mod:`!reprlib` module provides a means for producing object representations with limits on the size of the resulting strings. This is used in the Python debugger and may be useful in other contexts as well. @@ -58,29 +58,31 @@ This module provides a class, an instance, and a function: limits on most sizes. In addition to size-limiting tools, the module also provides a decorator for -detecting recursive calls to :meth:`__repr__` and substituting a placeholder -string instead. +detecting recursive calls to :meth:`~object.__repr__` and substituting a +placeholder string instead. .. index:: single: ...; placeholder .. decorator:: recursive_repr(fillvalue="...") - Decorator for :meth:`__repr__` methods to detect recursive calls within the + Decorator for :meth:`~object.__repr__` methods to detect recursive calls within the same thread. If a recursive call is made, the *fillvalue* is returned, - otherwise, the usual :meth:`__repr__` call is made. For example: - - >>> from reprlib import recursive_repr - >>> class MyList(list): - ... @recursive_repr() - ... def __repr__(self): - ... return '<' + '|'.join(map(repr, self)) + '>' - ... - >>> m = MyList('abc') - >>> m.append(m) - >>> m.append('x') - >>> print(m) - <'a'|'b'|'c'|...|'x'> + otherwise, the usual :meth:`!__repr__` call is made. For example: + + .. doctest:: + + >>> from reprlib import recursive_repr + >>> class MyList(list): + ... @recursive_repr() + ... def __repr__(self): + ... return '<' + '|'.join(map(repr, self)) + '>' + ... + >>> m = MyList('abc') + >>> m.append(m) + >>> m.append('x') + >>> print(m) + <'a'|'b'|'c'|...|'x'> .. versionadded:: 3.2 @@ -148,10 +150,10 @@ which format specific object types. with no line breaks or indentation, like the standard :func:`repr`. For example: - .. code-block:: pycon + .. doctest:: indent >>> example = [ - 1, 'spam', {'a': 2, 'b': 'spam eggs', 'c': {3: 4.5, 6: []}}, 'ham'] + ... 1, 'spam', {'a': 2, 'b': 'spam eggs', 'c': {3: 4.5, 6: []}}, 'ham'] >>> import reprlib >>> aRepr = reprlib.Repr() >>> print(aRepr.repr(example)) @@ -160,7 +162,7 @@ which format specific object types. If :attr:`~Repr.indent` is set to a string, each recursion level is placed on its own line, indented by that string: - .. code-block:: pycon + .. doctest:: indent >>> aRepr.indent = '-->' >>> print(aRepr.repr(example)) @@ -181,7 +183,7 @@ which format specific object types. Setting :attr:`~Repr.indent` to a positive integer value behaves as if it was set to a string with that number of spaces: - .. code-block:: pycon + .. doctest:: indent >>> aRepr.indent = 4 >>> print(aRepr.repr(example)) @@ -234,7 +236,9 @@ Subclassing Repr Objects The use of dynamic dispatching by :meth:`Repr.repr1` allows subclasses of :class:`Repr` to add support for additional built-in object types or to modify the handling of types already supported. This example shows how special support -for file objects could be added:: +for file objects could be added: + +.. testcode:: import reprlib import sys @@ -248,3 +252,7 @@ for file objects could be added:: aRepr = MyRepr() print(aRepr.repr(sys.stdin)) # prints '' + +.. testoutput:: + + diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index ef65674d1b0a78..4e58b043f1da31 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -277,7 +277,7 @@ These functions are used to retrieve resource usage information: This function returns an object that describes the resources consumed by either the current process or its children, as specified by the *who* parameter. The - *who* parameter should be specified using one of the :const:`RUSAGE_\*` + *who* parameter should be specified using one of the :const:`!RUSAGE_\*` constants described below. A simple example:: @@ -353,7 +353,7 @@ These functions are used to retrieve resource usage information: Returns the number of bytes in a system page. (This need not be the same as the hardware page size.) -The following :const:`RUSAGE_\*` symbols are passed to the :func:`getrusage` +The following :const:`!RUSAGE_\*` symbols are passed to the :func:`getrusage` function to specify which processes information should be provided for. diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 40b09ce897880e..8287699c5f013e 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -10,12 +10,14 @@ -------------- -The :mod:`rlcompleter` module defines a completion function suitable for the -:mod:`readline` module by completing valid Python identifiers and keywords. +The :mod:`!rlcompleter` module defines a completion function suitable to be +passed to :func:`~readline.set_completer` in the :mod:`readline` module. When this module is imported on a Unix platform with the :mod:`readline` module available, an instance of the :class:`Completer` class is automatically created -and its :meth:`complete` method is set as the :mod:`readline` completer. +and its :meth:`~Completer.complete` method is set as the +:ref:`readline completer `. The method provides +completion of valid Python :ref:`identifiers and keywords `. Example:: @@ -28,7 +30,7 @@ Example:: readline.__name__ readline.parse_and_bind( >>> readline. -The :mod:`rlcompleter` module is designed for use with Python's +The :mod:`!rlcompleter` module is designed for use with Python's :ref:`interactive mode `. Unless Python is run with the :option:`-S` option, the module is automatically imported and configured (see :ref:`rlcompleter-config`). @@ -39,23 +41,25 @@ this module can still be used for custom purposes. .. _completer-objects: -Completer Objects ------------------ +.. class:: Completer -Completer objects have the following method: + Completer objects have the following method: + .. method:: Completer.complete(text, state) -.. method:: Completer.complete(text, state) + Return the next possible completion for *text*. - Return the *state*\ th completion for *text*. + When called by the :mod:`readline` module, this method is called + successively with ``state == 0, 1, 2, ...`` until the method returns + ``None``. - If called for *text* that doesn't include a period character (``'.'``), it will - complete from names currently defined in :mod:`__main__`, :mod:`builtins` and - keywords (as defined by the :mod:`keyword` module). - - If called for a dotted name, it will try to evaluate anything without obvious - side-effects (functions will not be evaluated, but it can generate calls to - :meth:`__getattr__`) up to the last part, and find matches for the rest via the - :func:`dir` function. Any exception raised during the evaluation of the - expression is caught, silenced and :const:`None` is returned. + If called for *text* that doesn't include a period character (``'.'``), it will + complete from names currently defined in :mod:`__main__`, :mod:`builtins` and + keywords (as defined by the :mod:`keyword` module). + If called for a dotted name, it will try to evaluate anything without obvious + side-effects (functions will not be evaluated, but it can generate calls to + :meth:`~object.__getattr__`) up to the last part, and find matches for the + rest via the :func:`dir` function. Any exception raised during the + evaluation of the expression is caught, silenced and :const:`None` is + returned. diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 219219af6fd87f..95c54991887022 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -113,6 +113,9 @@ Restrictions differs across Unix versions and requires knowledge about the database implementation used. +* On macOS :mod:`dbm.ndbm` can silently corrupt the database file on updates, + which can cause hard crashes when trying to read from the database. + .. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') @@ -149,13 +152,14 @@ Restrictions .. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') - A subclass of :class:`Shelf` which exposes :meth:`first`, :meth:`!next`, - :meth:`previous`, :meth:`last` and :meth:`set_location` which are available - in the third-party :mod:`bsddb` module from `pybsddb + A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, + :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. + These are available + in the third-party :mod:`!bsddb` module from `pybsddb `_ but not in other database modules. The *dict* object passed to the constructor must support those methods. This is generally accomplished by calling one of - :func:`bsddb.hashopen`, :func:`bsddb.btopen` or :func:`bsddb.rnopen`. The + :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The optional *protocol*, *writeback*, and *keyencoding* parameters have the same interpretation as for the :class:`Shelf` class. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d1949d698f5614..f61ef8b0ecc7ba 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -343,6 +343,12 @@ Directory and files operations .. versionchanged:: 3.12 Added the *onexc* parameter, deprecated *onerror*. + .. versionchanged:: 3.13 + :func:`!rmtree` now ignores :exc:`FileNotFoundError` exceptions for all + but the top-level path. + Exceptions other than :exc:`OSError` and subclasses of :exc:`!OSError` + are now always propagated to the caller. + .. attribute:: rmtree.avoids_symlink_attacks Indicates whether the current platform and implementation provides a diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 7ee5ece8859825..85a073aad233ac 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -157,6 +157,8 @@ The variables defined in the :mod:`signal` module are: Alias to :data:`SIGCHLD`. + .. availability:: not macOS. + .. data:: SIGCONT Continue the process if it is currently stopped diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index e36fc17f89de24..4bfb0d8c2cfeac 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -185,7 +185,7 @@ created. Socket addresses are represented as follows: .. versionadded:: 3.7 - :const:`AF_PACKET` is a low-level interface directly to network devices. - The packets are represented by the tuple + The addresses are represented by the tuple ``(ifname, proto[, pkttype[, hatype[, addr]]])`` where: - *ifname* - String specifying the device name. @@ -193,7 +193,6 @@ created. Socket addresses are represented as follows: May be :data:`ETH_P_ALL` to capture all protocols, one of the :ref:`ETHERTYPE_* constants ` or any other Ethernet protocol number. - Value must be in network-byte-order. - *pkttype* - Optional integer specifying the packet type: - ``PACKET_HOST`` (the default) - Packet addressed to the local host. @@ -312,7 +311,7 @@ Exceptions The accompanying value is a pair ``(error, string)`` representing an error returned by a library call. *string* represents the description of *error*, as returned by the :c:func:`gai_strerror` C function. The - numeric *error* value will match one of the :const:`EAI_\*` constants + numeric *error* value will match one of the :const:`!EAI_\*` constants defined in this module. .. versionchanged:: 3.3 @@ -1518,7 +1517,7 @@ to sockets. .. method:: socket.getsockopt(level, optname[, buflen]) Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:const:`SO_\*` etc.) + :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) are defined in this module. If *buflen* is absent, an integer option is assumed and its integer value is returned by the function. If *buflen* is present, it specifies the maximum length of the buffer used to receive the option in, and @@ -1938,8 +1937,8 @@ to sockets. .. index:: pair: module; struct Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in the - :mod:`socket` module (:const:`SO_\*` etc.). The value can be an integer, + :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this + module (:ref:`!SO_\* etc. `). The value can be an integer, ``None`` or a :term:`bytes-like object` representing a buffer. In the later case it is up to the caller to ensure that the bytestring contains the proper bits (see the optional built-in module :mod:`struct` for a way to diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 206294528e0016..e8709b516ae07a 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -25,7 +25,7 @@ probably additional platforms, as long as OpenSSL is installed on that platform. Some behavior may be platform dependent, since calls are made to the operating system socket APIs. The installed version of OpenSSL may also - cause variations in behavior. For example, TLSv1.3 with OpenSSL version + cause variations in behavior. For example, TLSv1.3 comes with OpenSSL version 1.1.1. .. warning:: @@ -908,6 +908,12 @@ Constants .. versionadded:: 3.7 +.. data:: HAS_PSK + + Whether the OpenSSL library has built-in support for TLS-PSK. + + .. versionadded:: 3.13 + .. data:: CHANNEL_BINDING_TYPES List of supported TLS channel binding types. Strings in this list @@ -2050,6 +2056,9 @@ to speed up repeated connections from the same clients. return 'ClientId_1', psk_table.get(hint, b'') context.set_psk_client_callback(callback) + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + .. versionadded:: 3.13 .. method:: SSLContext.set_psk_server_callback(callback, identity_hint=None) @@ -2092,6 +2101,9 @@ to speed up repeated connections from the same clients. return psk_table.get(identity, b'') context.set_psk_server_callback(callback, 'ServerId_1') + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + .. versionadded:: 3.13 .. index:: single: certificates diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index f204b287b565eb..9028ff5c134fa9 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -44,7 +44,8 @@ Any object can be tested for truth value, for use in an :keyword:`if` or .. index:: single: true By default, an object is considered true unless its class defines either a -:meth:`~object.__bool__` method that returns ``False`` or a :meth:`__len__` method that +:meth:`~object.__bool__` method that returns ``False`` or a +:meth:`~object.__len__` method that returns zero, when called with the object. [1]_ Here are most of the built-in objects considered false: @@ -197,7 +198,7 @@ exception. Two more operations with the same syntactic priority, :keyword:`in` and :keyword:`not in`, are supported by types that are :term:`iterable` or -implement the :meth:`__contains__` method. +implement the :meth:`~object.__contains__` method. .. _typesnumeric: @@ -717,7 +718,7 @@ that's defined for any rational number, and hence applies to all instances of :class:`int` and :class:`fractions.Fraction`, and all finite instances of :class:`float` and :class:`decimal.Decimal`. Essentially, this function is given by reduction modulo ``P`` for a fixed prime ``P``. The value of ``P`` is -made available to Python as the :attr:`modulus` attribute of +made available to Python as the :attr:`~sys.hash_info.modulus` attribute of :data:`sys.hash_info`. .. impl-detail:: @@ -906,9 +907,9 @@ Generator Types --------------- Python's :term:`generator`\s provide a convenient way to implement the iterator -protocol. If a container object's :meth:`__iter__` method is implemented as a +protocol. If a container object's :meth:`~iterator.__iter__` method is implemented as a generator, it will automatically return an iterator object (technically, a -generator object) supplying the :meth:`__iter__` and :meth:`~generator.__next__` +generator object) supplying the :meth:`!__iter__` and :meth:`~generator.__next__` methods. More information about generators can be found in :ref:`the documentation for the yield expression `. @@ -3672,7 +3673,7 @@ The conversion types are: +------------+-----------------------------------------------------+-------+ | ``'b'`` | Bytes (any object that follows the | \(5) | | | :ref:`buffer protocol ` or has | | -| | :meth:`__bytes__`). | | +| | :meth:`~object.__bytes__`). | | +------------+-----------------------------------------------------+-------+ | ``'s'`` | ``'s'`` is an alias for ``'b'`` and should only | \(6) | | | be used for Python2/3 code bases. | | @@ -4410,7 +4411,8 @@ The constructors for both classes work the same: :meth:`symmetric_difference_update` methods will accept any iterable as an argument. - Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and + Note, the *elem* argument to the :meth:`~object.__contains__`, + :meth:`remove`, and :meth:`discard` methods may be a set. To support searching for an equivalent frozenset, a temporary one is created from *elem*. @@ -5236,9 +5238,11 @@ instantiated from the type:: TypeError: cannot create 'types.UnionType' instances .. note:: - The :meth:`__or__` method for type objects was added to support the syntax - ``X | Y``. If a metaclass implements :meth:`__or__`, the Union may - override it:: + The :meth:`!__or__` method for type objects was added to support the syntax + ``X | Y``. If a metaclass implements :meth:`!__or__`, the Union may + override it: + + .. doctest:: >>> class M(type): ... def __or__(self, other): @@ -5250,7 +5254,7 @@ instantiated from the type:: >>> C | int 'Hello' >>> int | C - int | __main__.C + int | C .. seealso:: @@ -5324,25 +5328,30 @@ Methods .. index:: pair: object; method Methods are functions that are called using the attribute notation. There are -two flavors: built-in methods (such as :meth:`append` on lists) and class -instance methods. Built-in methods are described with the types that support -them. +two flavors: :ref:`built-in methods ` (such as :meth:`append` +on lists) and :ref:`class instance method `. +Built-in methods are described with the types that support them. If you access a method (a function defined in a class namespace) through an instance, you get a special object: a :dfn:`bound method` (also called -:dfn:`instance method`) object. When called, it will add the ``self`` argument +:ref:`instance method `) object. When called, it will add +the ``self`` argument to the argument list. Bound methods have two special read-only attributes: -``m.__self__`` is the object on which the method operates, and ``m.__func__`` is +:attr:`m.__self__ ` is the object on which the method +operates, and :attr:`m.__func__ ` is the function implementing the method. Calling ``m(arg-1, arg-2, ..., arg-n)`` is completely equivalent to calling ``m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)``. -Like function objects, bound method objects support getting arbitrary +Like :ref:`function objects `, bound method objects support +getting arbitrary attributes. However, since method attributes are actually stored on the -underlying function object (``meth.__func__``), setting method attributes on +underlying function object (:attr:`method.__func__`), setting method attributes on bound methods is disallowed. Attempting to set an attribute on a method results in an :exc:`AttributeError` being raised. In order to set a method -attribute, you need to explicitly set it on the underlying function object:: +attribute, you need to explicitly set it on the underlying function object: + +.. doctest:: >>> class C: ... def method(self): @@ -5357,7 +5366,7 @@ attribute, you need to explicitly set it on the underlying function object:: >>> c.method.whoami 'my name is method' -See :ref:`types` for more information. +See :ref:`instance-methods` for more information. .. index:: object; code, code object @@ -5375,10 +5384,10 @@ Code objects are used by the implementation to represent "pseudo-compiled" executable Python code such as a function body. They differ from function objects because they don't contain a reference to their global execution environment. Code objects are returned by the built-in :func:`compile` function -and can be extracted from function objects through their :attr:`__code__` -attribute. See also the :mod:`code` module. +and can be extracted from function objects through their +:attr:`~function.__code__` attribute. See also the :mod:`code` module. -Accessing ``__code__`` raises an :ref:`auditing event ` +Accessing :attr:`~function.__code__` raises an :ref:`auditing event ` ``object.__getattr__`` with arguments ``obj`` and ``"__code__"``. .. index:: @@ -5452,8 +5461,9 @@ It is written as ``NotImplemented``. Internal Objects ---------------- -See :ref:`types` for this information. It describes stack frame objects, -traceback objects, and slice objects. +See :ref:`types` for this information. It describes +:ref:`stack frame objects `, +:ref:`traceback objects `, and slice objects. .. _specialattrs: diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index bf9aaca2a696de..2426c37ccb1e0f 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1205,6 +1205,18 @@ always available. .. versionadded:: 3.12 +.. function:: _is_interned(string) + + Return :const:`True` if the given string is "interned", :const:`False` + otherwise. + + .. versionadded:: 3.13 + + .. impl-detail:: + + It is not guaranteed to exist in all implementations of Python. + + .. data:: last_type last_value last_traceback @@ -1588,7 +1600,8 @@ always available. :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this works. Per-line events may be disabled for a frame by setting - :attr:`!f_trace_lines` to :const:`False` on that :ref:`frame `. + :attr:`~frame.f_trace_lines` to :const:`False` on that + :ref:`frame `. ``'return'`` A function (or other code block) is about to return. The local trace @@ -1606,7 +1619,7 @@ always available. opcode details). The local trace function is called; *arg* is ``None``; the return value specifies the new local trace function. Per-opcode events are not emitted by default: they must be explicitly - requested by setting :attr:`!f_trace_opcodes` to :const:`True` on the + requested by setting :attr:`~frame.f_trace_opcodes` to :const:`True` on the :ref:`frame `. Note that as an exception is propagated down the chain of callers, an @@ -1636,8 +1649,8 @@ always available. .. versionchanged:: 3.7 - ``'opcode'`` event type added; :attr:`!f_trace_lines` and - :attr:`!f_trace_opcodes` attributes added to frames + ``'opcode'`` event type added; :attr:`~frame.f_trace_lines` and + :attr:`~frame.f_trace_opcodes` attributes added to frames .. function:: set_asyncgen_hooks(firstiter, finalizer) @@ -1731,9 +1744,17 @@ always available. .. availability:: Windows. + .. note:: + Changing the filesystem encoding after Python startup is risky because + the old fsencoding or paths encoded by the old fsencoding may be cached + somewhere. Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. + .. versionadded:: 3.6 See :pep:`529` for more details. + .. deprecated-removed:: 3.13 3.16 + Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. + .. data:: stdin stdout stderr diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index b5ab446e0096ed..7b27fc7e85b62d 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -15,7 +15,7 @@ facility. This module wraps the system ``syslog`` family of routines. A pure Python library that can speak to a syslog server is available in the -:mod:`logging.handlers` module as :class:`SysLogHandler`. +:mod:`logging.handlers` module as :class:`~logging.handlers.SysLogHandler`. The module defines the following functions: @@ -107,22 +107,62 @@ The module defines the following functions: The module defines the following constants: -Priority levels (high to low): - :const:`LOG_EMERG`, :const:`LOG_ALERT`, :const:`LOG_CRIT`, :const:`LOG_ERR`, - :const:`LOG_WARNING`, :const:`LOG_NOTICE`, :const:`LOG_INFO`, - :const:`LOG_DEBUG`. - -Facilities: - :const:`LOG_KERN`, :const:`LOG_USER`, :const:`LOG_MAIL`, :const:`LOG_DAEMON`, - :const:`LOG_AUTH`, :const:`LOG_LPR`, :const:`LOG_NEWS`, :const:`LOG_UUCP`, - :const:`LOG_CRON`, :const:`LOG_SYSLOG`, :const:`LOG_LOCAL0` to - :const:`LOG_LOCAL7`, and, if defined in ````, - :const:`LOG_AUTHPRIV`. - -Log options: - :const:`LOG_PID`, :const:`LOG_CONS`, :const:`LOG_NDELAY`, and, if defined - in ````, :const:`LOG_ODELAY`, :const:`LOG_NOWAIT`, and - :const:`LOG_PERROR`. + +.. data:: LOG_EMERG + LOG_ALERT + LOG_CRIT + LOG_ERR + LOG_WARNING + LOG_NOTICE + LOG_INFO + LOG_DEBUG + + Priority levels (high to low). + + +.. data:: LOG_AUTH + LOG_AUTHPRIV + LOG_CRON + LOG_DAEMON + LOG_FTP + LOG_INSTALL + LOG_KERN + LOG_LAUNCHD + LOG_LPR + LOG_MAIL + LOG_NETINFO + LOG_NEWS + LOG_RAS + LOG_REMOTEAUTH + LOG_SYSLOG + LOG_USER + LOG_UUCP + LOG_LOCAL0 + LOG_LOCAL1 + LOG_LOCAL2 + LOG_LOCAL3 + LOG_LOCAL4 + LOG_LOCAL5 + LOG_LOCAL6 + LOG_LOCAL7 + + Facilities, depending on availability in ```` for :const:`LOG_AUTHPRIV`, + :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, + :const:`LOG_INSTALL` and :const:`LOG_RAS`. + + .. versionchanged:: 3.13 + Added :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, + :const:`LOG_INSTALL`, :const:`LOG_RAS`, and :const:`LOG_LAUNCHD`. + +.. data:: LOG_PID + LOG_CONS + LOG_NDELAY + LOG_ODELAY + LOG_NOWAIT + LOG_PERROR + + Log options, depending on availability in ```` for + :const:`LOG_ODELAY`, :const:`LOG_NOWAIT` and :const:`LOG_PERROR`. Examples diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 3e5723a66780ca..7ba29d4a40dedb 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -116,10 +116,12 @@ Some facts and figures: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will be done on the file. If given, *fileobj* may be any object that has a - :meth:`read` or :meth:`write` method (depending on the *mode*). *bufsize* - specifies the blocksize and defaults to ``20 * 512`` bytes. Use this variant - in combination with e.g. ``sys.stdin``, a socket :term:`file object` or a tape - device. However, such a :class:`TarFile` object is limited in that it does + :meth:`~io.RawIOBase.read` or :meth:`~io.RawIOBase.write` method + (depending on the *mode*) that works with bytes. + *bufsize* specifies the blocksize and defaults to ``20 * 512`` bytes. + Use this variant in combination with e.g. ``sys.stdin.buffer``, a socket + :term:`file object` or a tape device. + However, such a :class:`TarFile` object is limited in that it does not allow random access, see :ref:`tar-examples`. The currently possible modes: @@ -255,6 +257,51 @@ The following constants are available at the module level: The default character encoding: ``'utf-8'`` on Windows, the value returned by :func:`sys.getfilesystemencoding` otherwise. +.. data:: REGTYPE + AREGTYPE + + A regular file :attr:`~TarInfo.type`. + +.. data:: LNKTYPE + + A link (inside tarfile) :attr:`~TarInfo.type`. + +.. data:: SYMTYPE + + A symbolic link :attr:`~TarInfo.type`. + +.. data:: CHRTYPE + + A character special device :attr:`~TarInfo.type`. + +.. data:: BLKTYPE + + A block special device :attr:`~TarInfo.type`. + +.. data:: DIRTYPE + + A directory :attr:`~TarInfo.type`. + +.. data:: FIFOTYPE + + A FIFO special device :attr:`~TarInfo.type`. + +.. data:: CONTTYPE + + A contiguous file :attr:`~TarInfo.type`. + +.. data:: GNUTYPE_LONGNAME + + A GNU tar longname :attr:`~TarInfo.type`. + +.. data:: GNUTYPE_LONGLINK + + A GNU tar longlink :attr:`~TarInfo.type`. + +.. data:: GNUTYPE_SPARSE + + A GNU tar sparse file :attr:`~TarInfo.type`. + Each of the following constants defines a tar archive format that the :mod:`tarfile` module is able to create. See section :ref:`tar-formats` for @@ -325,7 +372,7 @@ be finalized; only the internally used file object will be closed. See the *name* is the pathname of the archive. *name* may be a :term:`path-like object`. It can be omitted if *fileobj* is given. - In this case, the file object's :attr:`name` attribute is used if it exists. + In this case, the file object's :attr:`!name` attribute is used if it exists. *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append data to an existing file, ``'w'`` to create a new file overwriting an existing @@ -359,7 +406,7 @@ be finalized; only the internally used file object will be closed. See the messages). The messages are written to ``sys.stderr``. *errorlevel* controls how extraction errors are handled, - see :attr:`the corresponding attribute <~TarFile.errorlevel>`. + see :attr:`the corresponding attribute `. The *encoding* and *errors* arguments define the character encoding to be used for reading or writing the archive and how conversion errors are going @@ -645,8 +692,8 @@ It does *not* contain the file's data itself. :meth:`~TarFile.getmember`, :meth:`~TarFile.getmembers` and :meth:`~TarFile.gettarinfo`. -Modifying the objects returned by :meth:`~!TarFile.getmember` or -:meth:`~!TarFile.getmembers` will affect all subsequent +Modifying the objects returned by :meth:`~TarFile.getmember` or +:meth:`~TarFile.getmembers` will affect all subsequent operations on the archive. For cases where this is unwanted, you can use :mod:`copy.copy() ` or call the :meth:`~TarInfo.replace` method to create a modified copy in one step. @@ -795,8 +842,8 @@ A ``TarInfo`` object has the following public data attributes: A dictionary containing key-value pairs of an associated pax extended header. -.. method:: TarInfo.replace(name=..., mtime=..., mode=..., linkname=..., - uid=..., gid=..., uname=..., gname=..., +.. method:: TarInfo.replace(name=..., mtime=..., mode=..., linkname=..., \ + uid=..., gid=..., uname=..., gname=..., \ deep=True) .. versionadded:: 3.12 @@ -816,7 +863,7 @@ A :class:`TarInfo` object also provides some convenient query methods: .. method:: TarInfo.isfile() - Return :const:`True` if the :class:`Tarinfo` object is a regular file. + Return :const:`True` if the :class:`TarInfo` object is a regular file. .. method:: TarInfo.isreg() @@ -952,7 +999,7 @@ reused in custom filters: path (after following symlinks) would end up outside the destination. This raises :class:`~tarfile.OutsideDestinationError`. - Clear high mode bits (setuid, setgid, sticky) and group/other write bits - (:const:`~stat.S_IWGRP`|:const:`~stat.S_IWOTH`). + (:const:`~stat.S_IWGRP` | :const:`~stat.S_IWOTH`). Return the modified ``TarInfo`` member. @@ -977,9 +1024,9 @@ reused in custom filters: - For regular files, including hard links: - Set the owner read and write permissions - (:const:`~stat.S_IRUSR`|:const:`~stat.S_IWUSR`). + (:const:`~stat.S_IRUSR` | :const:`~stat.S_IWUSR`). - Remove the group & other executable permission - (:const:`~stat.S_IXGRP`|:const:`~stat.S_IXOTH`) + (:const:`~stat.S_IXGRP` | :const:`~stat.S_IXOTH`) if the owner doesn’t have it (:const:`~stat.S_IXUSR`). - For other files (directories), set ``mode`` to ``None``, so diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index b68a78e8267bcc..9add8500c7788c 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -18,7 +18,7 @@ This module creates temporary files and directories. It works on all supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, :class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level interfaces which provide automatic cleanup and can be used as -context managers. :func:`mkstemp` and +:term:`context managers `. :func:`mkstemp` and :func:`mkdtemp` are lower-level functions which require manual cleanup. All the user-callable functions and constructors take additional arguments which @@ -41,7 +41,7 @@ The module defines the following user-callable items: this; your code should not rely on a temporary file created using this function having or not having a visible name in the file system. - The resulting object can be used as a context manager (see + The resulting object can be used as a :term:`context manager` (see :ref:`tempfile-examples`). On completion of the context or destruction of the file object the temporary file will be removed from the filesystem. @@ -87,9 +87,9 @@ The module defines the following user-callable items: determine whether and how the named file should be automatically deleted. The returned object is always a :term:`file-like object` whose :attr:`!file` - attribute is the underlying true file object. This :term:`file-like object` + attribute is the underlying true file object. This file-like object can be used in a :keyword:`with` statement, just like a normal file. The - name of the temporary file can be retrieved from the :attr:`name` attribute + name of the temporary file can be retrieved from the :attr:`!name` attribute of the returned file-like object. On Unix, unlike with the :func:`TemporaryFile`, the directory entry does not get unlinked immediately after the file creation. @@ -151,18 +151,20 @@ The module defines the following user-callable items: contents are written to disk and operation proceeds as with :func:`TemporaryFile`. - The resulting file has one additional method, :func:`rollover`, which - causes the file to roll over to an on-disk file regardless of its size. + .. method:: SpooledTemporaryFile.rollover - The returned object is a file-like object whose :attr:`_file` attribute + The resulting file has one additional method, :meth:`!rollover`, which + causes the file to roll over to an on-disk file regardless of its size. + + The returned object is a file-like object whose :attr:`!_file` attribute is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object (depending on whether binary or text *mode* was specified) or a true file - object, depending on whether :func:`rollover` has been called. This + object, depending on whether :meth:`rollover` has been called. This file-like object can be used in a :keyword:`with` statement, just like a normal file. .. versionchanged:: 3.3 - the truncate method now accepts a ``size`` argument. + the truncate method now accepts a *size* argument. .. versionchanged:: 3.8 Added *errors* parameter. @@ -176,24 +178,28 @@ The module defines the following user-callable items: .. class:: TemporaryDirectory(suffix=None, prefix=None, dir=None, ignore_cleanup_errors=False, *, delete=True) This class securely creates a temporary directory using the same rules as :func:`mkdtemp`. - The resulting object can be used as a context manager (see + The resulting object can be used as a :term:`context manager` (see :ref:`tempfile-examples`). On completion of the context or destruction of the temporary directory object, the newly created temporary directory and all its contents are removed from the filesystem. - The directory name can be retrieved from the :attr:`name` attribute of the - returned object. When the returned object is used as a context manager, the - :attr:`name` will be assigned to the target of the :keyword:`!as` clause in - the :keyword:`with` statement, if there is one. - - The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. If *ignore_cleanup_errors* is true, any unhandled - exceptions during explicit or implicit cleanup (such as a - :exc:`PermissionError` removing open files on Windows) will be ignored, - and the remaining removable items deleted on a "best-effort" basis. - Otherwise, errors will be raised in whatever context cleanup occurs - (the :func:`cleanup` call, exiting the context manager, when the object - is garbage-collected or during interpreter shutdown). + .. attribute:: TemporaryDirectory.name + + The directory name can be retrieved from the :attr:`!name` attribute of the + returned object. When the returned object is used as a :term:`context manager`, the + :attr:`!name` will be assigned to the target of the :keyword:`!as` clause in + the :keyword:`with` statement, if there is one. + + .. method:: TemporaryDirectory.cleanup + + The directory can be explicitly cleaned up by calling the + :meth:`!cleanup` method. If *ignore_cleanup_errors* is true, any unhandled + exceptions during explicit or implicit cleanup (such as a + :exc:`PermissionError` removing open files on Windows) will be ignored, + and the remaining removable items deleted on a "best-effort" basis. + Otherwise, errors will be raised in whatever context cleanup occurs + (the :meth:`!cleanup` call, exiting the context manager, when the object + is garbage-collected or during interpreter shutdown). The *delete* parameter can be used to disable cleanup of the directory tree upon exiting the context. While it may seem unusual for a context manager diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 6cbdc39b3c024d..9173db07fd0071 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1408,7 +1408,8 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. class:: FakePath(path) - Simple :term:`path-like object`. It implements the :meth:`__fspath__` + Simple :term:`path-like object`. It implements the + :meth:`~os.PathLike.__fspath__` method which just returns the *path* argument. If *path* is an exception, it will be raised in :meth:`!__fspath__`. diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index a8c67b02a23e4d..e084d8554c7c09 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -58,8 +58,8 @@ details that are unchanged. * `Modern Tkinter for Busy Python Developers `_ By Mark Roseman. (ISBN 978-1999149567) - * `Python and Tkinter Programming `_ - By Alan Moore. (ISBN 978-1788835886) + * `Python GUI programming with Tkinter `_ + By Alan D. Moore. (ISBN 978-1788835886) * `Programming Python `_ By Mark Lutz; has excellent coverage of Tkinter. (ISBN 978-0596158101) diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 5fab1454944989..1609dc2ce9218e 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -986,19 +986,19 @@ ttk.Treeview The valid options/values are: - id + *id* Returns the column name. This is a read-only option. - anchor: One of the standard Tk anchor values. + *anchor*: One of the standard Tk anchor values. Specifies how the text in this column should be aligned with respect to the cell. - minwidth: width + *minwidth*: width The minimum width of the column in pixels. The treeview widget will not make the column any smaller than specified by this option when the widget is resized or the user drags a column. - stretch: ``True``/``False`` + *stretch*: ``True``/``False`` Specifies whether the column's width should be adjusted when the widget is resized. - width: width + *width*: width The width of the column in pixels. To configure the tree column, call this with column = "#0" @@ -1041,14 +1041,14 @@ ttk.Treeview The valid options/values are: - text: text + *text*: text The text to display in the column heading. - image: imageName + *image*: imageName Specifies an image to display to the right of the column heading. - anchor: anchor + *anchor*: anchor Specifies how the heading text should be aligned. One of the standard Tk anchor values. - command: callback + *command*: callback A callback to be invoked when the heading label is pressed. To configure the tree column heading, call this with column = "#0". @@ -1391,7 +1391,8 @@ option. If you don't know the class name of a widget, use the method .. method:: element_create(elementname, etype, *args, **kw) Create a new element in the current theme, of the given *etype* which is - expected to be either "image" or "from". + expected to be either "image", "from" or "vsapi". + The latter is only available in Tk 8.6 on Windows. If "image" is used, *args* should contain the default image name followed by statespec/value pairs (this is the imagespec), and *kw* may have the @@ -1439,6 +1440,63 @@ option. If you don't know the class name of a widget, use the method style = ttk.Style(root) style.element_create('plain.background', 'from', 'default') + If "vsapi" is used as the value of *etype*, :meth:`element_create` + will create a new element in the current theme whose visual appearance + is drawn using the Microsoft Visual Styles API which is responsible + for the themed styles on Windows XP and Vista. + *args* is expected to contain the Visual Styles class and part as + given in the Microsoft documentation followed by an optional sequence + of tuples of ttk states and the corresponding Visual Styles API state + value. + *kw* may have the following options: + + padding=padding + Specify the element's interior padding. + *padding* is a list of up to four integers specifying the left, + top, right and bottom padding quantities respectively. + If fewer than four elements are specified, bottom defaults to top, + right defaults to left, and top defaults to left. + In other words, a list of three numbers specify the left, vertical, + and right padding; a list of two numbers specify the horizontal + and the vertical padding; a single number specifies the same + padding all the way around the widget. + This option may not be mixed with any other options. + + margins=padding + Specifies the elements exterior padding. + *padding* is a list of up to four integers specifying the left, top, + right and bottom padding quantities respectively. + This option may not be mixed with any other options. + + width=width + Specifies the width for the element. + If this option is set then the Visual Styles API will not be queried + for the recommended size or the part. + If this option is set then *height* should also be set. + The *width* and *height* options cannot be mixed with the *padding* + or *margins* options. + + height=height + Specifies the height of the element. + See the comments for *width*. + + Example:: + + style = ttk.Style(root) + style.element_create('pin', 'vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]) + style.layout('Explorer.Pin', + [('Explorer.Pin.pin', {'sticky': 'news'})]) + pin = ttk.Checkbutton(style='Explorer.Pin') + pin.pack(expand=True, fill='both') + + .. versionchanged:: 3.13 + Added support of the "vsapi" element factory. .. method:: element_names() @@ -1515,23 +1573,24 @@ Layouts A layout can be just ``None``, if it takes no options, or a dict of options specifying how to arrange the element. The layout mechanism uses a simplified version of the pack geometry manager: given an -initial cavity, each element is allocated a parcel. Valid -options/values are: +initial cavity, each element is allocated a parcel. + +The valid options/values are: -side: whichside +*side*: whichside Specifies which side of the cavity to place the element; one of top, right, bottom or left. If omitted, the element occupies the entire cavity. -sticky: nswe +*sticky*: nswe Specifies where the element is placed inside its allocated parcel. -unit: 0 or 1 +*unit*: 0 or 1 If set to 1, causes the element and all of its descendants to be treated as a single element for the purposes of :meth:`Widget.identify` et al. It's used for things like scrollbar thumbs with grips. -children: [sublayout... ] +*children*: [sublayout... ] Specifies a list of elements to place inside the element. Each element is a tuple (or other sequence type) where the first item is the layout name, and the other is a `Layout`_. diff --git a/Doc/library/tomllib.rst b/Doc/library/tomllib.rst index 918576eb37eaee..f9e2dfeb13dc87 100644 --- a/Doc/library/tomllib.rst +++ b/Doc/library/tomllib.rst @@ -95,7 +95,7 @@ Conversion Table +------------------+--------------------------------------------------------------------------------------+ | TOML | Python | +==================+======================================================================================+ -| table | dict | +| TOML document | dict | +------------------+--------------------------------------------------------------------------------------+ | string | str | +------------------+--------------------------------------------------------------------------------------+ @@ -115,3 +115,9 @@ Conversion Table +------------------+--------------------------------------------------------------------------------------+ | array | list | +------------------+--------------------------------------------------------------------------------------+ +| table | dict | ++------------------+--------------------------------------------------------------------------------------+ +| inline table | dict | ++------------------+--------------------------------------------------------------------------------------+ +| array of tables | list of dicts | ++------------------+--------------------------------------------------------------------------------------+ diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 408da7fc5f0645..ab83e0df10b709 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -16,8 +16,10 @@ interpreter. .. index:: pair: object; traceback -The module uses traceback objects --- these are objects of type :class:`types.TracebackType`, -which are assigned to the ``__traceback__`` field of :class:`BaseException` instances. +The module uses :ref:`traceback objects ` --- these are +objects of type :class:`types.TracebackType`, +which are assigned to the :attr:`~BaseException.__traceback__` field of +:class:`BaseException` instances. .. seealso:: @@ -31,11 +33,13 @@ The module defines the following functions: .. function:: print_tb(tb, limit=None, file=None) - Print up to *limit* stack trace entries from traceback object *tb* (starting + Print up to *limit* stack trace entries from + :ref:`traceback object ` *tb* (starting from the caller's frame) if *limit* is positive. Otherwise, print the last ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries are printed. If *file* is omitted or ``None``, the output goes to - ``sys.stderr``; otherwise it should be an open file or file-like object to + :data:`sys.stderr`; otherwise it should be an open + :term:`file ` or :term:`file-like object` to receive the output. .. versionchanged:: 3.5 @@ -45,7 +49,8 @@ The module defines the following functions: .. function:: print_exception(exc, /[, value, tb], limit=None, \ file=None, chain=True) - Print exception information and stack trace entries from traceback object + Print exception information and stack trace entries from + :ref:`traceback object ` *tb* to *file*. This differs from :func:`print_tb` in the following ways: @@ -66,7 +71,8 @@ The module defines the following functions: The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the - :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be + :attr:`~BaseException.__cause__` or :attr:`~BaseException.__context__` + attributes of the exception) will be printed as well, like the interpreter itself does when printing an unhandled exception. @@ -96,7 +102,8 @@ The module defines the following functions: Print up to *limit* stack trace entries (starting from the invocation point) if *limit* is positive. Otherwise, print the last ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries are printed. - The optional *f* argument can be used to specify an alternate stack frame + The optional *f* argument can be used to specify an alternate + :ref:`stack frame ` to start. The optional *file* argument has the same meaning as for :func:`print_tb`. @@ -107,20 +114,20 @@ The module defines the following functions: .. function:: extract_tb(tb, limit=None) Return a :class:`StackSummary` object representing a list of "pre-processed" - stack trace entries extracted from the traceback object *tb*. It is useful + stack trace entries extracted from the + :ref:`traceback object ` *tb*. It is useful for alternate formatting of stack traces. The optional *limit* argument has the same meaning as for :func:`print_tb`. A "pre-processed" stack trace entry is a :class:`FrameSummary` object containing attributes :attr:`~FrameSummary.filename`, :attr:`~FrameSummary.lineno`, :attr:`~FrameSummary.name`, and :attr:`~FrameSummary.line` representing the - information that is usually printed for a stack trace. The - :attr:`~FrameSummary.line` is a string with leading and trailing - whitespace stripped; if the source is not available it is ``None``. + information that is usually printed for a stack trace. .. function:: extract_stack(f=None, limit=None) - Extract the raw traceback from the current stack frame. The return value has + Extract the raw traceback from the current + :ref:`stack frame `. The return value has the same format as for :func:`extract_tb`. The optional *f* and *limit* arguments have the same meaning as for :func:`print_stack`. @@ -138,7 +145,7 @@ The module defines the following functions: .. function:: format_exception_only(exc, /[, value], *, show_group=False) Format the exception part of a traceback using an exception value such as - given by ``sys.last_value``. The return value is a list of strings, each + given by :data:`sys.last_value`. The return value is a list of strings, each ending in a newline. The list contains the exception's message, which is normally a single string; however, for :exc:`SyntaxError` exceptions, it contains several lines that (when printed) display detailed information @@ -158,7 +165,8 @@ The module defines the following functions: positional-only. .. versionchanged:: 3.11 - The returned list now includes any notes attached to the exception. + The returned list now includes any + :attr:`notes ` attached to the exception. .. versionchanged:: 3.13 *show_group* parameter was added. @@ -197,14 +205,17 @@ The module defines the following functions: .. function:: clear_frames(tb) - Clears the local variables of all the stack frames in a traceback *tb* - by calling the :meth:`clear` method of each frame object. + Clears the local variables of all the stack frames in a + :ref:`traceback ` *tb* + by calling the :meth:`~frame.clear` method of each + :ref:`frame object `. .. versionadded:: 3.4 .. function:: walk_stack(f) - Walk a stack following ``f.f_back`` from the given frame, yielding the frame + Walk a stack following :attr:`f.f_back ` from the given frame, + yielding the frame and line number for each frame. If *f* is ``None``, the current stack is used. This helper is used with :meth:`StackSummary.extract`. @@ -212,19 +223,20 @@ The module defines the following functions: .. function:: walk_tb(tb) - Walk a traceback following ``tb_next`` yielding the frame and line number + Walk a traceback following :attr:`~traceback.tb_next` yielding the frame and + line number for each frame. This helper is used with :meth:`StackSummary.extract`. .. versionadded:: 3.5 The module also defines the following classes: -:class:`TracebackException` Objects ------------------------------------ +:class:`!TracebackException` Objects +------------------------------------ .. versionadded:: 3.5 -:class:`TracebackException` objects are created from actual exceptions to +:class:`!TracebackException` objects are created from actual exceptions to capture data for later printing in a lightweight fashion. .. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) @@ -232,10 +244,11 @@ capture data for later printing in a lightweight fashion. Capture an exception for later rendering. *limit*, *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` class. - If *compact* is true, only data that is required by :class:`TracebackException`'s - ``format`` method is saved in the class attributes. In particular, the - ``__context__`` field is calculated only if ``__cause__`` is ``None`` and - ``__suppress_context__`` is false. + If *compact* is true, only data that is required by + :class:`!TracebackException`'s :meth:`format` method + is saved in the class attributes. In particular, the + :attr:`__context__` field is calculated only if :attr:`__cause__` is + ``None`` and :attr:`__suppress_context__` is false. Note that when locals are captured, they are also shown in the traceback. @@ -253,27 +266,31 @@ capture data for later printing in a lightweight fashion. .. attribute:: __cause__ - A :class:`TracebackException` of the original ``__cause__``. + A :class:`!TracebackException` of the original + :attr:`~BaseException.__cause__`. .. attribute:: __context__ - A :class:`TracebackException` of the original ``__context__``. + A :class:`!TracebackException` of the original + :attr:`~BaseException.__context__`. .. attribute:: exceptions If ``self`` represents an :exc:`ExceptionGroup`, this field holds a list of - :class:`TracebackException` instances representing the nested exceptions. + :class:`!TracebackException` instances representing the nested exceptions. Otherwise it is ``None``. .. versionadded:: 3.11 .. attribute:: __suppress_context__ - The ``__suppress_context__`` value from the original exception. + The :attr:`~BaseException.__suppress_context__` value from the original + exception. .. attribute:: __notes__ - The ``__notes__`` value from the original exception, or ``None`` + The :attr:`~BaseException.__notes__` value from the original exception, + or ``None`` if the exception does not have any notes. If it is not ``None`` is it formatted in the traceback after the exception string. @@ -287,6 +304,14 @@ capture data for later printing in a lightweight fashion. The class of the original traceback. + .. deprecated:: 3.13 + + .. attribute:: exc_type_str + + String display of the class of the original exception. + + .. versionadded:: 3.13 + .. attribute:: filename For syntax errors - the file name where the error occurred. @@ -339,8 +364,8 @@ capture data for later printing in a lightweight fashion. Format the exception. - If *chain* is not ``True``, ``__cause__`` and ``__context__`` will not - be formatted. + If *chain* is not ``True``, :attr:`__cause__` and :attr:`__context__` + will not be formatted. The return value is a generator of strings, each ending in a newline and some containing internal newlines. :func:`~traceback.print_exception` @@ -363,33 +388,34 @@ capture data for later printing in a lightweight fashion. well, recursively, with indentation relative to their nesting depth. .. versionchanged:: 3.11 - The exception's notes are now included in the output. + The exception's :attr:`notes ` are now + included in the output. .. versionchanged:: 3.13 Added the *show_group* parameter. -:class:`StackSummary` Objects ------------------------------ +:class:`!StackSummary` Objects +------------------------------ .. versionadded:: 3.5 -:class:`StackSummary` objects represent a call stack ready for formatting. +:class:`!StackSummary` objects represent a call stack ready for formatting. .. class:: StackSummary .. classmethod:: extract(frame_gen, *, limit=None, lookup_lines=True, capture_locals=False) - Construct a :class:`StackSummary` object from a frame generator (such as + Construct a :class:`!StackSummary` object from a frame generator (such as is returned by :func:`~traceback.walk_stack` or :func:`~traceback.walk_tb`). If *limit* is supplied, only this many frames are taken from *frame_gen*. If *lookup_lines* is ``False``, the returned :class:`FrameSummary` objects will not have read their lines in yet, making the cost of - creating the :class:`StackSummary` cheaper (which may be valuable if it + creating the :class:`!StackSummary` cheaper (which may be valuable if it may not actually get formatted). If *capture_locals* is ``True`` the - local variables in each :class:`FrameSummary` are captured as object + local variables in each :class:`!FrameSummary` are captured as object representations. .. versionchanged:: 3.12 @@ -398,14 +424,16 @@ capture data for later printing in a lightweight fashion. .. classmethod:: from_list(a_list) - Construct a :class:`StackSummary` object from a supplied list of + Construct a :class:`!StackSummary` object from a supplied list of :class:`FrameSummary` objects or old-style list of tuples. Each tuple - should be a 4-tuple with filename, lineno, name, line as the elements. + should be a 4-tuple with *filename*, *lineno*, *name*, *line* as the + elements. .. method:: format() Returns a list of strings ready for printing. Each string in the - resulting list corresponds to a single frame from the stack. + resulting list corresponds to a single :ref:`frame ` from + the stack. Each string ends in a newline; the strings may contain internal newlines as well, for those items with source text lines. @@ -418,7 +446,8 @@ capture data for later printing in a lightweight fashion. .. method:: format_frame_summary(frame_summary) - Returns a string for printing one of the frames involved in the stack. + Returns a string for printing one of the :ref:`frames ` + involved in the stack. This method is called for each :class:`FrameSummary` object to be printed by :meth:`StackSummary.format`. If it returns ``None``, the frame is omitted from the output. @@ -426,25 +455,50 @@ capture data for later printing in a lightweight fashion. .. versionadded:: 3.11 -:class:`FrameSummary` Objects ------------------------------ +:class:`!FrameSummary` Objects +------------------------------ .. versionadded:: 3.5 -A :class:`FrameSummary` object represents a single frame in a traceback. +A :class:`!FrameSummary` object represents a single :ref:`frame ` +in a :ref:`traceback `. .. class:: FrameSummary(filename, lineno, name, lookup_line=True, locals=None, line=None) - Represent a single frame in the traceback or stack that is being formatted - or printed. It may optionally have a stringified version of the frames + Represents a single :ref:`frame ` in the + :ref:`traceback ` or stack that is being formatted + or printed. It may optionally have a stringified version of the frame's locals included in it. If *lookup_line* is ``False``, the source code is not - looked up until the :class:`FrameSummary` has the :attr:`~FrameSummary.line` - attribute accessed (which also happens when casting it to a tuple). + looked up until the :class:`!FrameSummary` has the :attr:`~FrameSummary.line` + attribute accessed (which also happens when casting it to a :class:`tuple`). :attr:`~FrameSummary.line` may be directly provided, and will prevent line lookups happening at all. *locals* is an optional local variable dictionary, and if supplied the variable representations are stored in the summary for later display. + :class:`!FrameSummary` instances have the following attributes: + + .. attribute:: FrameSummary.filename + + The filename of the source code for this frame. Equivalent to accessing + :attr:`f.f_code.co_filename ` on a + :ref:`frame object ` *f*. + + .. attribute:: FrameSummary.lineno + + The line number of the source code for this frame. + + .. attribute:: FrameSummary.name + + Equivalent to accessing :attr:`f.f_code.co_name ` on + a :ref:`frame object ` *f*. + + .. attribute:: FrameSummary.line + + A string representing the source code for this frame, with leading and + trailing whitespace stripped. + If the source is not available, it is ``None``. + .. _traceback-example: Traceback Examples @@ -515,27 +569,32 @@ The output for the example would look similar to this: *** print_tb: File "", line 10, in lumberjack() + ~~~~~~~~~~^^ *** print_exception: Traceback (most recent call last): File "", line 10, in lumberjack() + ~~~~~~~~~~^^ File "", line 4, in lumberjack bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ IndexError: tuple index out of range *** print_exc: Traceback (most recent call last): File "", line 10, in lumberjack() + ~~~~~~~~~~^^ File "", line 4, in lumberjack bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ IndexError: tuple index out of range *** format_exc, first and last line: Traceback (most recent call last): IndexError: tuple index out of range *** format_exception: ['Traceback (most recent call last):\n', - ' File "", line 10, in \n lumberjack()\n', - ' File "", line 4, in lumberjack\n bright_side_of_life()\n', + ' File "", line 10, in \n lumberjack()\n ~~~~~~~~~~^^\n', + ' File "", line 4, in lumberjack\n bright_side_of_life()\n ~~~~~~~~~~~~~~~~~~~^^\n', ' File "", line 7, in bright_side_of_life\n return tuple()[0]\n ~~~~~~~^^^\n', 'IndexError: tuple index out of range\n'] *** extract_tb: @@ -543,8 +602,8 @@ The output for the example would look similar to this: , line 4 in lumberjack>, , line 7 in bright_side_of_life>] *** format_tb: - [' File "", line 10, in \n lumberjack()\n', - ' File "", line 4, in lumberjack\n bright_side_of_life()\n', + [' File "", line 10, in \n lumberjack()\n ~~~~~~~~~~^^\n', + ' File "", line 4, in lumberjack\n bright_side_of_life()\n ~~~~~~~~~~~~~~~~~~~^^\n', ' File "", line 7, in bright_side_of_life\n return tuple()[0]\n ~~~~~~~^^^\n'] *** tb_lineno: 10 diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 54c3907dec98cc..8ce67cf77253c3 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -378,17 +378,15 @@ Standard names are defined for the following types: .. data:: FrameType - The type of frame objects such as found in ``tb.tb_frame`` if ``tb`` is a - traceback object. - - See :ref:`the language reference ` for details of the - available attributes and operations. + The type of :ref:`frame objects ` such as found in + :attr:`tb.tb_frame ` if ``tb`` is a traceback object. .. data:: GetSetDescriptorType The type of objects defined in extension modules with ``PyGetSetDef``, such - as ``FrameType.f_locals`` or ``array.array.typecode``. This type is used as + as :attr:`FrameType.f_locals ` or ``array.array.typecode``. + This type is used as descriptor for object attributes; it has the same purpose as the :class:`property` type, but for classes defined in extension modules. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index ba2845eb17ddcc..63bd62d1f6679b 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2604,10 +2604,10 @@ Functions and decorators .. function:: reveal_type(obj, /) - Reveal the inferred static type of an expression. + Ask a static type checker to reveal the inferred type of an expression. When a static type checker encounters a call to this function, - it emits a diagnostic with the type of the argument. For example:: + it emits a diagnostic with the inferred type of the argument. For example:: x: int = 1 reveal_type(x) # Revealed type is "builtins.int" @@ -2615,22 +2615,21 @@ Functions and decorators This can be useful when you want to debug how your type checker handles a particular piece of code. - The function returns its argument unchanged, which allows using - it within an expression:: + At runtime, this function prints the runtime type of its argument to + :data:`sys.stderr` and returns the argument unchanged (allowing the call to + be used within an expression):: - x = reveal_type(1) # Revealed type is "builtins.int" + x = reveal_type(1) # prints "Runtime type is int" + print(x) # prints "1" + + Note that the runtime type may be different from (more or less specific + than) the type statically inferred by a type checker. Most type checkers support ``reveal_type()`` anywhere, even if the name is not imported from ``typing``. Importing the name from - ``typing`` allows your code to run without runtime errors and + ``typing``, however, allows your code to run without runtime errors and communicates intent more clearly. - At runtime, this function prints the runtime type of its argument to stderr - and returns it unchanged:: - - x = reveal_type(1) # prints "Runtime type is int" - print(x) # prints "1" - .. versionadded:: 3.11 .. decorator:: dataclass_transform(*, eq_default=True, order_default=False, \ diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index d6ac4d09300d9f..f1cc482c5cfe2a 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -824,8 +824,9 @@ apply to method calls on the mock object. .. class:: PropertyMock(*args, **kwargs) - A mock intended to be used as a property, or other descriptor, on a class. - :class:`PropertyMock` provides :meth:`__get__` and :meth:`__set__` methods + A mock intended to be used as a :class:`property`, or other + :term:`descriptor`, on a class. :class:`PropertyMock` provides + :meth:`~object.__get__` and :meth:`~object.__set__` methods so you can specify a return value when it is fetched. Fetching a :class:`PropertyMock` instance from an object calls the mock, with @@ -1707,8 +1708,9 @@ Keywords can be used in the :func:`patch.dict` call to set values in the diction :func:`patch.dict` can be used with dictionary like objects that aren't actually dictionaries. At the very minimum they must support item getting, setting, deleting and either iteration or membership test. This corresponds to the -magic methods :meth:`~object.__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either -:meth:`__iter__` or :meth:`__contains__`. +magic methods :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, +:meth:`~object.__delitem__` and either :meth:`~container.__iter__` or +:meth:`~object.__contains__`. >>> class Container: ... def __init__(self): @@ -2171,7 +2173,7 @@ For example: >>> object() in mock False -The two equality methods, :meth:`__eq__` and :meth:`__ne__`, are special. +The two equality methods, :meth:`!__eq__` and :meth:`!__ne__`, are special. They do the default equality comparison on identity, using the :attr:`~Mock.side_effect` attribute, unless you change their return value to return something else:: @@ -2521,8 +2523,8 @@ mock_open *read_data* is now reset on each call to the *mock*. .. versionchanged:: 3.8 - Added :meth:`__iter__` to implementation so that iteration (such as in for - loops) correctly consumes *read_data*. + Added :meth:`~container.__iter__` to implementation so that iteration + (such as in for loops) correctly consumes *read_data*. Using :func:`open` as a context manager is a great way to ensure your file handles are closed properly and is becoming common:: @@ -2704,7 +2706,7 @@ able to use autospec. On the other hand it is much better to design your objects so that introspection is safe [#]_. A more serious problem is that it is common for instance attributes to be -created in the :meth:`__init__` method and not to exist on the class at all. +created in the :meth:`~object.__init__` method and not to exist on the class at all. *autospec* can't know about any dynamically created attributes and restricts the api to visible attributes. :: @@ -2745,8 +2747,9 @@ this particular scenario: AttributeError: Mock object has no attribute 'a' Probably the best way of solving the problem is to add class attributes as -default values for instance members initialised in :meth:`__init__`. Note that if -you are only setting default attributes in :meth:`__init__` then providing them via +default values for instance members initialised in :meth:`~object.__init__`. +Note that if +you are only setting default attributes in :meth:`!__init__` then providing them via class attributes (shared between instances of course) is faster too. e.g. .. code-block:: python diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 02b72cb9f6b8aa..70b4c84c05f818 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -346,8 +346,8 @@ the `load_tests protocol`_. ``python -m unittest discover -s root/namespace -t root``). .. versionchanged:: 3.11 - Python 3.11 dropped the :term:`namespace packages ` - support. It has been broken since Python 3.7. Start directory and + :mod:`unittest` dropped the :term:`namespace packages ` + support in Python 3.11. It has been broken since Python 3.7. Start directory and subdirectories containing tests must be regular package that have ``__init__.py`` file. @@ -390,8 +390,8 @@ testing code:: widget = Widget('The widget') self.assertEqual(widget.size(), (50, 50)) -Note that in order to test something, we use one of the :meth:`assert\*` -methods provided by the :class:`TestCase` base class. If the test fails, an +Note that in order to test something, we use one of the :ref:`assert\* methods ` +provided by the :class:`TestCase` base class. If the test fails, an exception will be raised with an explanatory message, and :mod:`unittest` will identify the test case as a :dfn:`failure`. Any other exceptions will be treated as :dfn:`errors`. @@ -1733,7 +1733,7 @@ Grouping tests .. method:: __iter__() Tests grouped by a :class:`TestSuite` are always accessed by iteration. - Subclasses can lazily provide tests by overriding :meth:`__iter__`. Note + Subclasses can lazily provide tests by overriding :meth:`!__iter__`. Note that this method may be called several times on a single suite (for example when counting tests or comparing for equality) so the tests returned by repeated iterations before :meth:`TestSuite.run` must be the @@ -1744,7 +1744,7 @@ Grouping tests .. versionchanged:: 3.2 In earlier versions the :class:`TestSuite` accessed tests directly rather - than through iteration, so overriding :meth:`__iter__` wasn't sufficient + than through iteration, so overriding :meth:`!__iter__` wasn't sufficient for providing tests. .. versionchanged:: 3.4 @@ -1940,14 +1940,14 @@ Loading and running tests String giving the prefix of method names which will be interpreted as test methods. The default value is ``'test'``. - This affects :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` + This affects :meth:`getTestCaseNames` and all the ``loadTestsFrom*`` methods. .. attribute:: sortTestMethodsUsing Function to be used to compare method names when sorting them in - :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` methods. + :meth:`getTestCaseNames` and all the ``loadTestsFrom*`` methods. .. attribute:: suiteClass @@ -1956,7 +1956,7 @@ Loading and running tests methods on the resulting object are needed. The default value is the :class:`TestSuite` class. - This affects all the :meth:`loadTestsFrom\*` methods. + This affects all the ``loadTestsFrom*`` methods. .. attribute:: testNamePatterns @@ -1969,7 +1969,7 @@ Loading and running tests so unlike patterns passed to the ``-k`` option, simple substring patterns will have to be converted using ``*`` wildcards. - This affects all the :meth:`loadTestsFrom\*` methods. + This affects all the ``loadTestsFrom*`` methods. .. versionadded:: 3.7 @@ -2003,7 +2003,7 @@ Loading and running tests A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :ref:`assert\* methods `. .. attribute:: skipped diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index bf3af1bef0714c..0e18db73280a63 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -21,6 +21,14 @@ authentication, redirections, cookies and more. The `Requests package `_ is recommended for a higher-level HTTP client interface. +.. warning:: + + On macOS it is unsafe to use this module in programs using + :func:`os.fork` because the :func:`getproxies` implementation for + macOS uses a higher-level system API. Set the environment variable + ``no_proxy`` to ``*`` to avoid this problem + (e.g. ``os.environ["no_proxy"] = "*"``). + .. include:: ../includes/wasm-notavail.rst The :mod:`urllib.request` module defines the following functions: @@ -712,8 +720,8 @@ The following attribute and methods should only be used by classes derived from .. note:: The convention has been adopted that subclasses defining - :meth:`_request` or :meth:`_response` methods are named - :class:`\*Processor`; all others are named :class:`\*Handler`. + :meth:`!_request` or :meth:`!_response` methods are named + :class:`!\*Processor`; all others are named :class:`!\*Handler`. .. attribute:: BaseHandler.parent @@ -833,9 +841,9 @@ HTTPRedirectHandler Objects .. method:: HTTPRedirectHandler.redirect_request(req, fp, code, msg, hdrs, newurl) Return a :class:`Request` or ``None`` in response to a redirect. This is called - by the default implementations of the :meth:`http_error_30\*` methods when a + by the default implementations of the :meth:`!http_error_30\*` methods when a redirection is received from the server. If a redirection should take place, - return a new :class:`Request` to allow :meth:`http_error_30\*` to perform the + return a new :class:`Request` to allow :meth:`!http_error_30\*` to perform the redirect to *newurl*. Otherwise, raise :exc:`~urllib.error.HTTPError` if no other handler should try to handle this URL, or return ``None`` if you can't but another handler might. diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 884de08eab1b16..a9c469707e8227 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -522,6 +522,56 @@ Available Functions and calls to :func:`simplefilter`. +.. decorator:: deprecated(msg, *, category=DeprecationWarning, stacklevel=1) + + Decorator to indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, + deprecation warnings may be emitted at runtime when the object is used. + :term:`static type checkers ` + will also generate a diagnostic on usage of the deprecated object. + + Usage:: + + from warnings import deprecated + from typing import overload + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the :func:`@overload ` decorator + for the attribute to exist on the overload as returned by + :func:`typing.get_overloads`. + + .. versionadded:: 3.13 + See :pep:`702`. + + Available Context Managers -------------------------- diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index be9e56b04c1fbf..c2b0ba7046967e 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -201,8 +201,9 @@ manipulation of WSGI response headers using a mapping-like interface. an empty list. :class:`Headers` objects support typical mapping operations including - :meth:`~object.__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, - :meth:`__delitem__` and :meth:`__contains__`. For each of + :meth:`~object.__getitem__`, :meth:`~dict.get`, :meth:`~object.__setitem__`, + :meth:`~dict.setdefault`, + :meth:`~object.__delitem__` and :meth:`~object.__contains__`. For each of these methods, the key is the header name (treated case-insensitively), and the value is the first value associated with that header name. Setting a header deletes any existing values for that header, then adds a new value at the end of @@ -520,8 +521,10 @@ input, output, and error streams. want to subclass this instead of :class:`BaseCGIHandler`. This class is a subclass of :class:`BaseHandler`. It overrides the - :meth:`__init__`, :meth:`get_stdin`, :meth:`get_stderr`, :meth:`add_cgi_vars`, - :meth:`_write`, and :meth:`_flush` methods to support explicitly setting the + :meth:`!__init__`, :meth:`~BaseHandler.get_stdin`, + :meth:`~BaseHandler.get_stderr`, :meth:`~BaseHandler.add_cgi_vars`, + :meth:`~BaseHandler._write`, and :meth:`~BaseHandler._flush` methods to + support explicitly setting the environment and streams via the constructor. The supplied environment and streams are stored in the :attr:`stdin`, :attr:`stdout`, :attr:`stderr`, and :attr:`environ` attributes. diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index b387240a3716cc..d0e1b248d595d1 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -734,7 +734,7 @@ NamedNodeMap Objects attribute node. Get its value with the :attr:`value` attribute. There are also experimental methods that give this class more mapping behavior. -You can use them or you can use the standardized :meth:`getAttribute\*` family +You can use them or you can use the standardized :meth:`!getAttribute\*` family of methods on the :class:`Element` objects. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 146c4fd768233b..f7f23007fb0522 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -269,8 +269,9 @@ DateTime Objects Write the XML-RPC encoding of this :class:`DateTime` item to the *out* stream object. - It also supports certain of Python's built-in operators through rich comparison - and :meth:`__repr__` methods. + It also supports certain of Python's built-in operators through + :meth:`rich comparison ` and :meth:`~object.__repr__` + methods. A working example follows. The server code:: @@ -334,8 +335,8 @@ Binary Objects which was the de facto standard base64 specification when the XML-RPC spec was written. - It also supports certain of Python's built-in operators through :meth:`__eq__` - and :meth:`__ne__` methods. + It also supports certain of Python's built-in operators through + :meth:`~object.__eq__` and :meth:`~object.__ne__` methods. Example usage of the binary objects. We're going to transfer an image over XMLRPC:: diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 016369d2b89d2c..ca1ea455f0acfc 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -84,12 +84,12 @@ alone XML-RPC servers. Register a function that can respond to XML-RPC requests. If *name* is given, it will be the method name associated with *function*, otherwise - ``function.__name__`` will be used. *name* is a string, and may contain + :attr:`function.__name__` will be used. *name* is a string, and may contain characters not legal in Python identifiers, including the period character. This method can also be used as a decorator. When used as a decorator, *name* can only be given as a keyword argument to register *function* under - *name*. If no *name* is given, ``function.__name__`` will be used. + *name*. If no *name* is given, :attr:`function.__name__` will be used. .. versionchanged:: 3.7 :meth:`register_function` can be used as a decorator. @@ -298,12 +298,12 @@ requests sent to Python CGI scripts. Register a function that can respond to XML-RPC requests. If *name* is given, it will be the method name associated with *function*, otherwise - ``function.__name__`` will be used. *name* is a string, and may contain + :attr:`function.__name__` will be used. *name* is a string, and may contain characters not legal in Python identifiers, including the period character. This method can also be used as a decorator. When used as a decorator, *name* can only be given as a keyword argument to register *function* under - *name*. If no *name* is given, ``function.__name__`` will be used. + *name*. If no *name* is given, :attr:`function.__name__` will be used. .. versionchanged:: 3.7 :meth:`register_function` can be used as a decorator. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 8f6481339837a0..374404bf33abbe 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1261,7 +1261,8 @@ except that the original function is not temporarily bound to the name ``func``. A list of :ref:`type parameters ` may be given in square brackets between the function's name and the opening parenthesis for its parameter list. This indicates to static type checkers that the function is generic. At runtime, -the type parameters can be retrieved from the function's ``__type_params__`` +the type parameters can be retrieved from the function's +:attr:`~function.__type_params__` attribute. See :ref:`generic-functions` for more. .. versionchanged:: 3.12 @@ -1361,12 +1362,15 @@ access the local variables of the function containing the def. See section :pep:`526` - Syntax for Variable Annotations Ability to type hint variable declarations, including class - variables and instance variables + variables and instance variables. :pep:`563` - Postponed Evaluation of Annotations Support for forward references within annotations by preserving annotations in a string form at runtime instead of eager evaluation. + :pep:`318` - Decorators for Functions and Methods + Function and method decorators were introduced. + Class decorators were introduced in :pep:`3129`. .. _class: @@ -1868,8 +1872,8 @@ like ``TYPE_PARAMS_OF_ListOrSet`` are not actually bound at runtime. are mappings. .. [#] A string literal appearing as the first statement in the function body is - transformed into the function's ``__doc__`` attribute and therefore the - function's :term:`docstring`. + transformed into the function's :attr:`~function.__doc__` attribute and + therefore the function's :term:`docstring`. .. [#] A string literal appearing as the first statement in the class body is transformed into the namespace's ``__doc__`` item and therefore the class's diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index f7d3d2d0bbec23..d611bda298b509 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -88,7 +88,7 @@ Some objects contain references to "external" resources such as open files or windows. It is understood that these resources are freed when the object is garbage-collected, but since garbage collection is not guaranteed to happen, such objects also provide an explicit way to release the external resource, -usually a :meth:`close` method. Programs are strongly recommended to explicitly +usually a :meth:`!close` method. Programs are strongly recommended to explicitly close such objects. The ':keyword:`try`...\ :keyword:`finally`' statement and the ':keyword:`with`' statement provide convenient ways to do this. @@ -519,6 +519,8 @@ These are the types to which the function call operation (see section :ref:`calls`) can be applied: +.. _user-defined-funcs: + User-defined functions ^^^^^^^^^^^^^^^^^^^^^^ @@ -532,9 +534,34 @@ section :ref:`function`). It should be called with an argument list containing the same number of items as the function's formal parameter list. -Special attributes: +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: __closure__ (function attribute) + single: __globals__ (function attribute) + pair: global; namespace + +.. list-table:: + :header-rows: 1 + + * - Attribute + - Meaning + + * - .. attribute:: function.__globals__ + - A reference to the :class:`dictionary ` that holds the function's + :ref:`global variables ` -- the global namespace of the module + in which the function was defined. + + * - .. attribute:: function.__closure__ + - ``None`` or a :class:`tuple` of cells that contain bindings for the + function's free variables. + + A cell object has the attribute ``cell_contents``. + This can be used to get the value of the cell, as well as set the value. -.. tabularcolumns:: |l|L|l| +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. index:: single: __doc__ (function attribute) @@ -542,97 +569,84 @@ Special attributes: single: __module__ (function attribute) single: __dict__ (function attribute) single: __defaults__ (function attribute) - single: __closure__ (function attribute) single: __code__ (function attribute) - single: __globals__ (function attribute) single: __annotations__ (function attribute) single: __kwdefaults__ (function attribute) single: __type_params__ (function attribute) - pair: global; namespace -+-------------------------+-------------------------------+-----------+ -| Attribute | Meaning | | -+=========================+===============================+===========+ -| :attr:`__doc__` | The function's documentation | Writable | -| | string, or ``None`` if | | -| | unavailable; not inherited by | | -| | subclasses. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~definition.\ | The function's name. | Writable | -| __name__` | | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~definition.\ | The function's | Writable | -| __qualname__` | :term:`qualified name`. | | -| | | | -| | .. versionadded:: 3.3 | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__module__` | The name of the module the | Writable | -| | function was defined in, or | | -| | ``None`` if unavailable. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__defaults__` | A tuple containing default | Writable | -| | argument values for those | | -| | arguments that have defaults, | | -| | or ``None`` if no arguments | | -| | have a default value. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__code__` | The code object representing | Writable | -| | the compiled function body. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__globals__` | A reference to the dictionary | Read-only | -| | that holds the function's | | -| | global variables --- the | | -| | global namespace of the | | -| | module in which the function | | -| | was defined. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~object.__dict__`| The namespace supporting | Writable | -| | arbitrary function | | -| | attributes. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__closure__` | ``None`` or a tuple of cells | Read-only | -| | that contain bindings for the | | -| | function's free variables. | | -| | See below for information on | | -| | the ``cell_contents`` | | -| | attribute. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__annotations__` | A dict containing annotations | Writable | -| | of parameters. The keys of | | -| | the dict are the parameter | | -| | names, and ``'return'`` for | | -| | the return annotation, if | | -| | provided. For more | | -| | information on working with | | -| | this attribute, see | | -| | :ref:`annotations-howto`. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__kwdefaults__` | A dict containing defaults | Writable | -| | for keyword-only parameters. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__type_params__` | A tuple containing the | Writable | -| | :ref:`type parameters | | -| | ` of a | | -| | :ref:`generic function | | -| | `. | | -+-------------------------+-------------------------------+-----------+ - -Most of the attributes labelled "Writable" check the type of the assigned value. +Most of these attributes check the type of the assigned value: + +.. list-table:: + :header-rows: 1 + + * - Attribute + - Meaning + + * - .. attribute:: function.__doc__ + - The function's documentation string, or ``None`` if unavailable. + Not inherited by subclasses. + + * - .. attribute:: function.__name__ + - The function's name. + See also: :attr:`__name__ attributes `. + + * - .. attribute:: function.__qualname__ + - The function's :term:`qualified name`. + See also: :attr:`__qualname__ attributes `. + + .. versionadded:: 3.3 + + * - .. attribute:: function.__module__ + - The name of the module the function was defined in, + or ``None`` if unavailable. + + * - .. attribute:: function.__defaults__ + - A :class:`tuple` containing default :term:`parameter` values + for those parameters that have defaults, + or ``None`` if no parameters have a default value. + + * - .. attribute:: function.__code__ + - The :ref:`code object ` representing + the compiled function body. + + * - .. attribute:: function.__dict__ + - The namespace supporting arbitrary function attributes. + See also: :attr:`__dict__ attributes `. + + * - .. attribute:: function.__annotations__ + - A :class:`dictionary ` containing annotations of + :term:`parameters `. + The keys of the dictionary are the parameter names, + and ``'return'`` for the return annotation, if provided. + See also: :ref:`annotations-howto`. + + * - .. attribute:: function.__kwdefaults__ + - A :class:`dictionary ` containing defaults for keyword-only + :term:`parameters `. + + * - .. attribute:: function.__type_params__ + - A :class:`tuple` containing the :ref:`type parameters ` of + a :ref:`generic function `. + + .. versionadded:: 3.12 Function objects also support getting and setting arbitrary attributes, which can be used, for example, to attach metadata to functions. Regular attribute -dot-notation is used to get and set such attributes. *Note that the current -implementation only supports function attributes on user-defined functions. -Function attributes on built-in functions may be supported in the future.* +dot-notation is used to get and set such attributes. + +.. impl-detail:: -A cell object has the attribute ``cell_contents``. This can be used to get -the value of the cell, as well as set the value. + CPython's current implementation only supports function attributes + on user-defined functions. Function attributes on + :ref:`built-in functions ` may be supported in the + future. Additional information about a function's definition can be retrieved from its -code object; see the description of internal types below. The -:data:`cell ` type can be accessed in the :mod:`types` -module. +:ref:`code object ` +(accessible via the :attr:`~function.__code__` attribute). + +.. _instance-methods: Instance methods ^^^^^^^^^^^^^^^^ @@ -652,43 +666,66 @@ callable object (normally a user-defined function). single: __name__ (method attribute) single: __module__ (method attribute) -Special read-only attributes: :attr:`__self__` is the class instance object, -:attr:`__func__` is the function object; :attr:`__doc__` is the method's -documentation (same as ``__func__.__doc__``); :attr:`~definition.__name__` is the -method name (same as ``__func__.__name__``); :attr:`__module__` is the -name of the module the method was defined in, or ``None`` if unavailable. +Special read-only attributes: + +.. list-table:: + + * - .. attribute:: method.__self__ + - Refers to the class instance object to which the method is + :ref:`bound ` + + * - .. attribute:: method.__func__ + - Refers to the original :ref:`function object ` + + * - .. attribute:: method.__doc__ + - The method's documentation + (same as :attr:`method.__func__.__doc__ `). + A :class:`string ` if the original function had a docstring, else + ``None``. + + * - .. attribute:: method.__name__ + - The name of the method + (same as :attr:`method.__func__.__name__ `) + + * - .. attribute:: method.__module__ + - The name of the module the method was defined in, or ``None`` if + unavailable. Methods also support accessing (but not setting) the arbitrary function -attributes on the underlying function object. +attributes on the underlying :ref:`function object `. User-defined method objects may be created when getting an attribute of a class (perhaps via an instance of that class), if that attribute is a -user-defined function object or a class method object. +user-defined :ref:`function object ` or a +:class:`classmethod` object. + +.. _method-binding: When an instance method object is created by retrieving a user-defined -function object from a class via one of its instances, its -:attr:`__self__` attribute is the instance, and the method object is said -to be bound. The new method's :attr:`__func__` attribute is the original -function object. - -When an instance method object is created by retrieving a class method -object from a class or instance, its :attr:`__self__` attribute is the -class itself, and its :attr:`__func__` attribute is the function object +:ref:`function object ` from a class via one of its +instances, its :attr:`~method.__self__` attribute is the instance, and the +method object is said to be *bound*. The new method's :attr:`~method.__func__` +attribute is the original function object. + +When an instance method object is created by retrieving a :class:`classmethod` +object from a class or instance, its :attr:`~method.__self__` attribute is the +class itself, and its :attr:`~method.__func__` attribute is the function object underlying the class method. When an instance method object is called, the underlying function -(:attr:`__func__`) is called, inserting the class instance -(:attr:`__self__`) in front of the argument list. For instance, when -:class:`C` is a class which contains a definition for a function -:meth:`f`, and ``x`` is an instance of :class:`C`, calling ``x.f(1)`` is +(:attr:`~method.__func__`) is called, inserting the class instance +(:attr:`~method.__self__`) in front of the argument list. For instance, when +:class:`!C` is a class which contains a definition for a function +:meth:`!f`, and ``x`` is an instance of :class:`!C`, calling ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``. -When an instance method object is derived from a class method object, the -"class instance" stored in :attr:`__self__` will actually be the class +When an instance method object is derived from a :class:`classmethod` object, the +"class instance" stored in :attr:`~method.__self__` will actually be the class itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is the underlying function. -Note that the transformation from function object to instance method +Note that the transformation from :ref:`function object ` +to instance method object happens each time the attribute is retrieved from the instance. In some cases, a fruitful optimization is to assign the attribute to a local variable and call that local variable. Also notice that this @@ -754,6 +791,8 @@ is raised and the asynchronous iterator will have reached the end of the set of values to be yielded. +.. _builtin-functions: + Built-in functions ^^^^^^^^^^^^^^^^^^ @@ -766,11 +805,17 @@ A built-in function object is a wrapper around a C function. Examples of built-in functions are :func:`len` and :func:`math.sin` (:mod:`math` is a standard built-in module). The number and type of the arguments are determined by the C function. Special read-only attributes: -:attr:`__doc__` is the function's documentation string, or ``None`` if -unavailable; :attr:`~definition.__name__` is the function's name; :attr:`__self__` is -set to ``None`` (but see the next item); :attr:`__module__` is the name of -the module the function was defined in or ``None`` if unavailable. +* :attr:`!__doc__` is the function's documentation string, or ``None`` if + unavailable. See :attr:`function.__doc__`. +* :attr:`!__name__` is the function's name. See :attr:`function.__name__`. +* :attr:`!__self__` is set to ``None`` (but see the next item). +* :attr:`!__module__` is the name of + the module the function was defined in or ``None`` if unavailable. + See :attr:`function.__module__`. + + +.. _builtin-methods: Built-in methods ^^^^^^^^^^^^^^^^ @@ -783,8 +828,9 @@ Built-in methods This is really a different disguise of a built-in function, this time containing an object passed to the C function as an implicit extra argument. An example of a built-in method is ``alist.append()``, assuming *alist* is a list object. In -this case, the special read-only attribute :attr:`__self__` is set to the object -denoted by *alist*. +this case, the special read-only attribute :attr:`!__self__` is set to the object +denoted by *alist*. (The attribute has the same semantics as it does with +:attr:`other instance methods `.) Classes @@ -793,7 +839,7 @@ Classes Classes are callable. These objects normally act as factories for new instances of themselves, but variations are possible for class types that override :meth:`~object.__new__`. The arguments of the call are passed to -:meth:`__new__` and, in the typical case, to :meth:`~object.__init__` to +:meth:`!__new__` and, in the typical case, to :meth:`~object.__init__` to initialize the new instance. @@ -816,7 +862,8 @@ the :ref:`import system ` as invoked either by the :keyword:`import` statement, or by calling functions such as :func:`importlib.import_module` and built-in :func:`__import__`. A module object has a namespace implemented by a -dictionary object (this is the dictionary referenced by the ``__globals__`` +:class:`dictionary ` object (this is the dictionary referenced by the +:attr:`~function.__globals__` attribute of functions defined in the module). Attribute references are translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object does not contain the code object used @@ -897,10 +944,11 @@ https://www.python.org/download/releases/2.3/mro/. pair: object; dictionary pair: class; attribute -When a class attribute reference (for class :class:`C`, say) would yield a +When a class attribute reference (for class :class:`!C`, say) would yield a class method object, it is transformed into an instance method object whose -:attr:`__self__` attribute is :class:`C`. When it would yield a static -method object, it is transformed into the object wrapped by the static method +:attr:`~method.__self__` attribute is :class:`!C`. +When it would yield a :class:`staticmethod` object, +it is transformed into the object wrapped by the static method object. See section :ref:`descriptors` for another way in which attributes retrieved from a class may differ from those actually contained in its :attr:`~object.__dict__`. @@ -968,7 +1016,7 @@ in which attribute references are searched. When an attribute is not found there, and the instance's class has an attribute by that name, the search continues with the class attributes. If a class attribute is found that is a user-defined function object, it is transformed into an instance method -object whose :attr:`__self__` attribute is the instance. Static method and +object whose :attr:`~method.__self__` attribute is the instance. Static method and class method objects are also transformed; see above under "Classes". See section :ref:`descriptors` for another way in which attributes of a class retrieved via its instances may differ from the objects actually stored in @@ -1075,57 +1123,111 @@ indirectly) to mutable objects. single: co_freevars (code object attribute) single: co_qualname (code object attribute) -Special read-only attributes: :attr:`co_name` gives the function name; -:attr:`co_qualname` gives the fully qualified function name; -:attr:`co_argcount` is the total number of positional arguments -(including positional-only arguments and arguments with default values); -:attr:`co_posonlyargcount` is the number of positional-only arguments -(including arguments with default values); :attr:`co_kwonlyargcount` is -the number of keyword-only arguments (including arguments with default -values); :attr:`co_nlocals` is the number of local variables used by the -function (including arguments); :attr:`co_varnames` is a tuple containing -the names of the local variables (starting with the argument names); -:attr:`co_cellvars` is a tuple containing the names of local variables -that are referenced by nested functions; :attr:`co_freevars` is a tuple -containing the names of free variables; :attr:`co_code` is a string -representing the sequence of bytecode instructions; :attr:`co_consts` is -a tuple containing the literals used by the bytecode; :attr:`co_names` is -a tuple containing the names used by the bytecode; :attr:`co_filename` is -the filename from which the code was compiled; :attr:`co_firstlineno` is -the first line number of the function; :attr:`co_lnotab` is a string -encoding the mapping from bytecode offsets to line numbers (for details -see the source code of the interpreter, is deprecated since 3.12 -and may be removed in 3.14); :attr:`co_stacksize` is the -required stack size; :attr:`co_flags` is an integer encoding a number -of flags for the interpreter. +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + + * - .. attribute:: codeobject.co_name + - The function name + + * - .. attribute:: codeobject.co_qualname + - The fully qualified function name + + * - .. attribute:: codeobject.co_argcount + - The total number of positional :term:`parameters ` + (including positional-only parameters and parameters with default values) + that the function has + + * - .. attribute:: codeobject.co_posonlyargcount + - The number of positional-only :term:`parameters ` + (including arguments with default values) that the function has + + * - .. attribute:: codeobject.co_kwonlyargcount + - The number of keyword-only :term:`parameters ` + (including arguments with default values) that the function has + + * - .. attribute:: codeobject.co_nlocals + - The number of :ref:`local variables ` used by the function + (including parameters) + + * - .. attribute:: codeobject.co_varnames + - A :class:`tuple` containing the names of the local variables in the + function (starting with the parameter names) + + * - .. attribute:: codeobject.co_cellvars + - A :class:`tuple` containing the names of :ref:`local variables ` + that are referenced by nested functions inside the function + + * - .. attribute:: codeobject.co_freevars + - A :class:`tuple` containing the names of free variables in the function + + * - .. attribute:: codeobject.co_code + - A string representing the sequence of :term:`bytecode` instructions in + the function + + * - .. attribute:: codeobject.co_consts + - A :class:`tuple` containing the literals used by the :term:`bytecode` in + the function + + * - .. attribute:: codeobject.co_names + - A :class:`tuple` containing the names used by the :term:`bytecode` in + the function + + * - .. attribute:: codeobject.co_filename + - The name of the file from which the code was compiled + + * - .. attribute:: codeobject.co_firstlineno + - The line number of the first line of the function + + * - .. attribute:: codeobject.co_lnotab + - A string encoding the mapping from :term:`bytecode` offsets to line + numbers. For details, see the source code of the interpreter. + + .. deprecated:: 3.12 + This attribute of code objects is deprecated, and may be removed in + Python 3.14. + + * - .. attribute:: codeobject.co_stacksize + - The required stack size of the code object + + * - .. attribute:: codeobject.co_flags + - An :class:`integer ` encoding a number of flags for the + interpreter. .. index:: pair: object; generator -The following flag bits are defined for :attr:`co_flags`: bit ``0x04`` is set if +The following flag bits are defined for :attr:`~codeobject.co_flags`: +bit ``0x04`` is set if the function uses the ``*arguments`` syntax to accept an arbitrary number of positional arguments; bit ``0x08`` is set if the function uses the ``**keywords`` syntax to accept arbitrary keyword arguments; bit ``0x20`` is set -if the function is a generator. +if the function is a generator. See :ref:`inspect-module-co-flags` for details +on the semantics of each flags that might be present. Future feature declarations (``from __future__ import division``) also use bits -in :attr:`co_flags` to indicate whether a code object was compiled with a +in :attr:`~codeobject.co_flags` to indicate whether a code object was compiled with a particular feature enabled: bit ``0x2000`` is set if the function was compiled with future division enabled; bits ``0x10`` and ``0x1000`` were used in earlier versions of Python. -Other bits in :attr:`co_flags` are reserved for internal use. +Other bits in :attr:`~codeobject.co_flags` are reserved for internal use. .. index:: single: documentation string -If a code object represents a function, the first item in :attr:`co_consts` is +If a code object represents a function, the first item in +:attr:`~codeobject.co_consts` is the documentation string of the function, or ``None`` if undefined. +Methods on code objects +~~~~~~~~~~~~~~~~~~~~~~~ + .. method:: codeobject.co_positions() - Returns an iterable over the source code positions of each bytecode + Returns an iterable over the source code positions of each :term:`bytecode` instruction in the code object. - The iterator returns tuples containing the ``(start_line, end_line, + The iterator returns :class:`tuple`\s containing the ``(start_line, end_line, start_column, end_column)``. The *i-th* tuple corresponds to the position of the source code that compiled to the *i-th* instruction. Column information is 0-indexed utf-8 byte offsets on the given source @@ -1153,6 +1255,41 @@ the documentation string of the function, or ``None`` if undefined. :option:`-X` ``no_debug_ranges`` command line flag or the :envvar:`PYTHONNODEBUGRANGES` environment variable can be used. +.. method:: codeobject.co_lines() + + Returns an iterator that yields information about successive ranges of + :term:`bytecode`\s. Each item yielded is a ``(start, end, lineno)`` + :class:`tuple`: + + * ``start`` (an :class:`int`) represents the offset (inclusive) of the start + of the :term:`bytecode` range + * ``end`` (an :class:`int`) represents the offset (inclusive) of the end of + the :term:`bytecode` range + * ``lineno`` is an :class:`int` representing the line number of the + :term:`bytecode` range, or ``None`` if the bytecodes in the given range + have no line number + + The items yielded generated will have the following properties: + + * The first range yielded will have a ``start`` of 0. + * The ``(start, end)`` ranges will be non-decreasing and consecutive. That + is, for any pair of :class:`tuple`\s, the ``start`` of the second will be + equal to the ``end`` of the first. + * No range will be backwards: ``end >= start`` for all triples. + * The :class:`tuple` yielded will have ``end`` equal to the size of the + :term:`bytecode`. + + Zero-width ranges, where ``start == end``, are allowed. Zero-width ranges + are used for lines that are present in the source code, but have been + eliminated by the :term:`bytecode` compiler. + + .. versionadded:: 3.10 + + .. seealso:: + + :pep:`626` - Precise line numbers for debugging and other tools. + The PEP that introduced the :meth:`!co_lines` method. + .. _frame-objects: @@ -1161,8 +1298,9 @@ Frame objects .. index:: pair: object; frame -Frame objects represent execution frames. They may occur in traceback objects -(see below), and are also passed to registered trace functions. +Frame objects represent execution frames. They may occur in +:ref:`traceback objects `, +and are also passed to registered trace functions. .. index:: single: f_back (frame attribute) @@ -1172,16 +1310,36 @@ Frame objects represent execution frames. They may occur in traceback objects single: f_lasti (frame attribute) single: f_builtins (frame attribute) -Special read-only attributes: :attr:`f_back` is to the previous stack frame -(towards the caller), or ``None`` if this is the bottom stack frame; -:attr:`f_code` is the code object being executed in this frame; :attr:`f_locals` -is the dictionary used to look up local variables; :attr:`f_globals` is used for -global variables; :attr:`f_builtins` is used for built-in (intrinsic) names; -:attr:`f_lasti` gives the precise instruction (this is an index into the -bytecode string of the code object). +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + + * - .. attribute:: frame.f_back + - Points to the previous stack frame (towards the caller), + or ``None`` if this is the bottom stack frame + + * - .. attribute:: frame.f_code + - The :ref:`code object ` being executed in this frame. + Accessing this attribute raises an :ref:`auditing event ` + ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. + + * - .. attribute:: frame.f_locals + - The dictionary used by the frame to look up + :ref:`local variables ` + + * - .. attribute:: frame.f_globals + - The dictionary used by the frame to look up + :ref:`global variables ` -Accessing ``f_code`` raises an :ref:`auditing event ` -``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. + * - .. attribute:: frame.f_builtins + - The dictionary used by the frame to look up + :ref:`built-in (intrinsic) names ` + + * - .. attribute:: frame.f_lasti + - The "precise instruction" of the frame object + (this is an index into the :term:`bytecode` string of the + :ref:`code object `) .. index:: single: f_trace (frame attribute) @@ -1189,30 +1347,44 @@ Accessing ``f_code`` raises an :ref:`auditing event ` single: f_trace_opcodes (frame attribute) single: f_lineno (frame attribute) -Special writable attributes: :attr:`f_trace`, if not ``None``, is a function -called for various events during code execution (this is used by the debugger). -Normally an event is triggered for each new source line - this can be -disabled by setting :attr:`f_trace_lines` to :const:`False`. +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + + * - .. attribute:: frame.f_trace + - If not ``None``, this is a function called for various events during + code execution (this is used by debuggers). Normally an event is + triggered for each new source line (see :attr:`~frame.f_trace_lines`). -Implementations *may* allow per-opcode events to be requested by setting -:attr:`f_trace_opcodes` to :const:`True`. Note that this may lead to -undefined interpreter behaviour if exceptions raised by the trace -function escape to the function being traced. + * - .. attribute:: frame.f_trace_lines + - Set this attribute to :const:`False` to disable triggering a tracing + event for each source line. -:attr:`f_lineno` is the current line number of the frame --- writing to this -from within a trace function jumps to the given line (only for the bottom-most -frame). A debugger can implement a Jump command (aka Set Next Statement) -by writing to f_lineno. + * - .. attribute:: frame.f_trace_opcodes + - Set this attribute to :const:`True` to allow per-opcode events to be + requested. Note that this may lead to + undefined interpreter behaviour if exceptions raised by the trace + function escape to the function being traced. + + * - .. attribute:: frame.f_lineno + - The current line number of the frame -- writing to this + from within a trace function jumps to the given line (only for the bottom-most + frame). A debugger can implement a Jump command (aka Set Next Statement) + by writing to this attribute. + +Frame object methods +~~~~~~~~~~~~~~~~~~~~ Frame objects support one method: .. method:: frame.clear() - This method clears all references to local variables held by the - frame. Also, if the frame belonged to a generator, the generator + This method clears all references to :ref:`local variables ` held by the + frame. Also, if the frame belonged to a :term:`generator`, the generator is finalized. This helps break reference cycles involving frame - objects (for example when catching an exception and storing its - traceback for later use). + objects (for example when catching an :ref:`exception ` + and storing its :ref:`traceback ` for later use). :exc:`RuntimeError` is raised if the frame is currently executing or suspended. @@ -1240,26 +1412,31 @@ Traceback objects single: sys.exception single: sys.last_traceback -Traceback objects represent a stack trace of an exception. A traceback object +Traceback objects represent the stack trace of an :ref:`exception `. +A traceback object is implicitly created when an exception occurs, and may also be explicitly created by calling :class:`types.TracebackType`. +.. versionchanged:: 3.7 + Traceback objects can now be explicitly instantiated from Python code. + For implicitly created tracebacks, when the search for an exception handler unwinds the execution stack, at each unwound level a traceback object is inserted in front of the current traceback. When an exception handler is entered, the stack trace is made available to the program. (See section :ref:`try`.) It is accessible as the third item of the -tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute +tuple returned by :func:`sys.exc_info`, and as the +:attr:`~BaseException.__traceback__` attribute of the caught exception. When the program contains no suitable handler, the stack trace is written (nicely formatted) to the standard error stream; if the interpreter is interactive, it is also made available to the user -as ``sys.last_traceback``. +as :data:`sys.last_traceback`. For explicitly created tracebacks, it is up to the creator of the traceback -to determine how the ``tb_next`` attributes should be linked to form a -full stack trace. +to determine how the :attr:`~traceback.tb_next` attributes should be linked to +form a full stack trace. .. index:: single: tb_frame (traceback attribute) @@ -1268,27 +1445,40 @@ full stack trace. pair: statement; try Special read-only attributes: -:attr:`tb_frame` points to the execution frame of the current level; -:attr:`tb_lineno` gives the line number where the exception occurred; -:attr:`tb_lasti` indicates the precise instruction. + +.. list-table:: + + * - .. attribute:: traceback.tb_frame + - Points to the execution :ref:`frame ` of the current + level. + + Accessing this attribute raises an + :ref:`auditing event ` ``object.__getattr__`` with arguments + ``obj`` and ``"tb_frame"``. + + * - .. attribute:: traceback.tb_lineno + - Gives the line number where the exception occurred + + * - .. attribute:: traceback.tb_lasti + - Indicates the "precise instruction". + The line number and last instruction in the traceback may differ from the -line number of its frame object if the exception occurred in a +line number of its :ref:`frame object ` if the exception +occurred in a :keyword:`try` statement with no matching except clause or with a -finally clause. - -Accessing ``tb_frame`` raises an :ref:`auditing event ` -``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. +:keyword:`finally` clause. .. index:: single: tb_next (traceback attribute) -Special writable attribute: :attr:`tb_next` is the next level in the stack -trace (towards the frame where the exception occurred), or ``None`` if -there is no next level. +.. attribute:: traceback.tb_next -.. versionchanged:: 3.7 - Traceback objects can now be explicitly instantiated from Python code, - and the ``tb_next`` attribute of existing instances can be updated. + The special writable attribute :attr:`!tb_next` is the next level in the + stack trace (towards the frame where the exception occurred), or ``None`` if + there is no next level. + + .. versionchanged:: 3.7 + This attribute is now writable Slice objects @@ -1756,7 +1946,8 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. .. note:: This method may still be bypassed when looking up special methods as the - result of implicit invocation via language syntax or built-in functions. + result of implicit invocation via language syntax or + :ref:`built-in functions `. See :ref:`special-lookup`. .. audit-event:: object.__getattr__ obj,name object.__getattribute__ @@ -1901,13 +2092,17 @@ class' :attr:`~object.__dict__`. Called to delete the attribute on an instance *instance* of the owner class. +Instances of descriptors may also have the :attr:`!__objclass__` attribute +present: + +.. attribute:: object.__objclass__ -The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module -as specifying the class where this object was defined (setting this -appropriately can assist in runtime introspection of dynamic class attributes). -For callables, it may indicate that an instance of the given type (or a -subclass) is expected or required as the first positional argument (for example, -CPython sets this attribute for unbound methods that are implemented in C). + The attribute :attr:`!__objclass__` is interpreted by the :mod:`inspect` module + as specifying the class where this object was defined (setting this + appropriately can assist in runtime introspection of dynamic class attributes). + For callables, it may indicate that an instance of the given type (or a + subclass) is expected or required as the first positional argument (for example, + CPython sets this attribute for unbound methods that are implemented in C). .. _descriptor-invocation: @@ -1988,13 +2183,14 @@ For instance bindings, the precedence of descriptor invocation depends on which descriptor methods are defined. A descriptor can define any combination of :meth:`~object.__get__`, :meth:`~object.__set__` and :meth:`~object.__delete__`. If it does not -define :meth:`__get__`, then accessing the attribute will return the descriptor +define :meth:`!__get__`, then accessing the attribute will return the descriptor object itself unless there is a value in the object's instance dictionary. If -the descriptor defines :meth:`__set__` and/or :meth:`__delete__`, it is a data +the descriptor defines :meth:`!__set__` and/or :meth:`!__delete__`, it is a data descriptor; if it defines neither, it is a non-data descriptor. Normally, data -descriptors define both :meth:`__get__` and :meth:`__set__`, while non-data -descriptors have just the :meth:`__get__` method. Data descriptors with -:meth:`__get__` and :meth:`__set__` (and/or :meth:`__delete__`) defined always override a redefinition in an +descriptors define both :meth:`!__get__` and :meth:`!__set__`, while non-data +descriptors have just the :meth:`!__get__` method. Data descriptors with +:meth:`!__get__` and :meth:`!__set__` (and/or :meth:`!__delete__`) defined +always override a redefinition in an instance dictionary. In contrast, non-data descriptors can be overridden by instances. @@ -2571,16 +2767,17 @@ either to emulate a sequence or to emulate a mapping; the difference is that for a sequence, the allowable keys should be the integers *k* for which ``0 <= k < N`` where *N* is the length of the sequence, or :class:`slice` objects, which define a range of items. It is also recommended that mappings provide the methods -:meth:`keys`, :meth:`values`, :meth:`items`, :meth:`get`, :meth:`clear`, -:meth:`setdefault`, :meth:`pop`, :meth:`popitem`, :meth:`!copy`, and -:meth:`update` behaving similar to those for Python's standard :class:`dictionary ` +:meth:`!keys`, :meth:`!values`, :meth:`!items`, :meth:`!get`, :meth:`!clear`, +:meth:`!setdefault`, :meth:`!pop`, :meth:`!popitem`, :meth:`!copy`, and +:meth:`!update` behaving similar to those for Python's standard :class:`dictionary ` objects. The :mod:`collections.abc` module provides a :class:`~collections.abc.MutableMapping` :term:`abstract base class` to help create those methods from a base set of -:meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__delitem__`, and :meth:`keys`. -Mutable sequences should provide methods :meth:`append`, :meth:`count`, -:meth:`index`, :meth:`extend`, :meth:`insert`, :meth:`pop`, :meth:`remove`, -:meth:`reverse` and :meth:`sort`, like Python standard :class:`list` +:meth:`~object.__getitem__`, :meth:`~object.__setitem__`, +:meth:`~object.__delitem__`, and :meth:`!keys`. +Mutable sequences should provide methods :meth:`!append`, :meth:`!count`, +:meth:`!index`, :meth:`!extend`, :meth:`!insert`, :meth:`!pop`, :meth:`!remove`, +:meth:`!reverse` and :meth:`!sort`, like Python standard :class:`list` objects. Finally, sequence types should implement addition (meaning concatenation) and multiplication (meaning repetition) by defining the methods @@ -2593,7 +2790,7 @@ operator; for mappings, ``in`` should search the mapping's keys; for sequences, it should search through the values. It is further recommended that both mappings and sequences implement the :meth:`~object.__iter__` method to allow efficient iteration -through the container; for mappings, :meth:`__iter__` should iterate +through the container; for mappings, :meth:`!__iter__` should iterate through the object's keys; for sequences, it should iterate through the values. .. method:: object.__len__(self) @@ -2646,10 +2843,10 @@ through the object's keys; for sequences, it should iterate through the values. .. method:: object.__getitem__(self, key) Called to implement evaluation of ``self[key]``. For :term:`sequence` types, - the accepted keys should be integers and slice objects. Note that the - special interpretation of negative indexes (if the class wishes to emulate a - :term:`sequence` type) is up to the :meth:`__getitem__` method. If *key* is - of an inappropriate type, :exc:`TypeError` may be raised; if of a value + the accepted keys should be integers. Optionally, they may support + :class:`slice` objects as well. Negative index support is also optional. + If *key* is + of an inappropriate type, :exc:`TypeError` may be raised; if *key* is a value outside the set of indexes for the sequence (after any special interpretation of negative values), :exc:`IndexError` should be raised. For :term:`mapping` types, if *key* is missing (not in the container), @@ -3172,7 +3369,7 @@ generators, coroutines do not directly support iteration. to the :meth:`~generator.send` method of the iterator that caused the coroutine to suspend. The result (return value, :exc:`StopIteration`, or other exception) is the same as when - iterating over the :meth:`__await__` return value, described above. + iterating over the :meth:`!__await__` return value, described above. .. method:: coroutine.throw(value) coroutine.throw(type[, value[, traceback]]) diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index bc1db7b039cd5a..b9cca4444c9141 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -1,3 +1,5 @@ +.. _full-grammar-specification: + Full Grammar specification ========================== diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 3e07d16068a627..0adfb0365934e4 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -708,10 +708,12 @@ and formatted string literals may be concatenated with plain string literals. single: ! (exclamation); in formatted string literal single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals + .. _f-strings: +.. _formatted-string-literals: -Formatted string literals -------------------------- +f-strings +--------- .. versionadded:: 3.6 diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index a9e65be1eda340..04132c78ce77a6 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -214,7 +214,7 @@ Assignment of an object to a single target is recursively defined as follows. object. This can either replace an existing key/value pair with the same key value, or insert a new key/value pair (if no key with the same value existed). - For user-defined objects, the :meth:`__setitem__` method is called with + For user-defined objects, the :meth:`~object.__setitem__` method is called with appropriate arguments. .. index:: pair: slicing; assignment @@ -351,7 +351,7 @@ If the right hand side is present, an annotated assignment performs the actual assignment before evaluating annotations (where applicable). If the right hand side is not present for an expression target, then the interpreter evaluates the target except for the last -:meth:`__setitem__` or :meth:`__setattr__` call. +:meth:`~object.__setitem__` or :meth:`~object.__setattr__` call. .. seealso:: @@ -578,7 +578,7 @@ The :dfn:`type` of the exception is the exception instance's class, the .. index:: pair: object; traceback A traceback object is normally created automatically when an exception is raised -and attached to it as the :attr:`__traceback__` attribute, which is writable. +and attached to it as the :attr:`~BaseException.__traceback__` attribute. You can create an exception and set your own traceback in one step using the :meth:`~BaseException.with_traceback` exception method (which returns the same exception instance, with its traceback set to its argument), like so:: @@ -592,11 +592,13 @@ same exception instance, with its traceback set to its argument), like so:: The ``from`` clause is used for exception chaining: if given, the second *expression* must be another exception class or instance. If the second expression is an exception instance, it will be attached to the raised -exception as the :attr:`__cause__` attribute (which is writable). If the +exception as the :attr:`~BaseException.__cause__` attribute (which is writable). If the expression is an exception class, the class will be instantiated and the resulting exception instance will be attached to the raised exception as the -:attr:`__cause__` attribute. If the raised exception is not handled, both -exceptions will be printed:: +:attr:`!__cause__` attribute. If the raised exception is not handled, both +exceptions will be printed: + +.. code-block:: pycon >>> try: ... print(1 / 0) @@ -605,19 +607,24 @@ exceptions will be printed:: ... Traceback (most recent call last): File "", line 2, in + print(1 / 0) + ~~^~~ ZeroDivisionError: division by zero The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 4, in + raise RuntimeError("Something bad happened") from exc RuntimeError: Something bad happened A similar mechanism works implicitly if a new exception is raised when an exception is already being handled. An exception may be handled when an :keyword:`except` or :keyword:`finally` clause, or a :keyword:`with` statement, is used. The previous exception is then -attached as the new exception's :attr:`__context__` attribute:: +attached as the new exception's :attr:`~BaseException.__context__` attribute: + +.. code-block:: pycon >>> try: ... print(1 / 0) @@ -626,16 +633,21 @@ attached as the new exception's :attr:`__context__` attribute:: ... Traceback (most recent call last): File "", line 2, in + print(1 / 0) + ~~^~~ ZeroDivisionError: division by zero During handling of the above exception, another exception occurred: Traceback (most recent call last): File "", line 4, in + raise RuntimeError("Something bad happened") RuntimeError: Something bad happened Exception chaining can be explicitly suppressed by specifying :const:`None` in -the ``from`` clause:: +the ``from`` clause: + +.. doctest:: >>> try: ... print(1 / 0) @@ -653,8 +665,8 @@ and information about handling exceptions is in section :ref:`try`. :const:`None` is now permitted as ``Y`` in ``raise X from Y``. .. versionadded:: 3.3 - The ``__suppress_context__`` attribute to suppress automatic display of the - exception context. + The :attr:`~BaseException.__suppress_context__` attribute to suppress + automatic display of the exception context. .. versionchanged:: 3.11 If the traceback of the active exception is modified in an :keyword:`except` @@ -920,7 +932,7 @@ That is not a future statement; it's an ordinary import statement with no special semantics or syntax restrictions. Code compiled by calls to the built-in functions :func:`exec` and :func:`compile` -that occur in a module :mod:`M` containing a future statement will, by default, +that occur in a module :mod:`!M` containing a future statement will, by default, use the new syntax or semantics associated with the future statement. This can be controlled by optional arguments to :func:`compile` --- see the documentation of that function for details. diff --git a/Doc/requirements.txt b/Doc/requirements.txt index ce87be2d392824..04334fd5a464d4 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -13,6 +13,7 @@ blurb sphinx-autobuild sphinxext-opengraph==0.7.5 +sphinx-notfound-page==1.0.0 # The theme used by the documentation is stored separately, so we need # to install that as well. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 026205d12d0e00..1e5fad18e91aee 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -23,24 +23,16 @@ Doc/extending/extending.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst -Doc/howto/isolating-extensions.rst Doc/howto/logging.rst Doc/howto/urllib2.rst -Doc/library/abc.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/asyncio-policy.rst Doc/library/asyncio-subprocess.rst Doc/library/asyncio-task.rst Doc/library/bdb.rst -Doc/library/bisect.rst -Doc/library/calendar.rst -Doc/library/cmd.rst -Doc/library/collections.abc.rst Doc/library/collections.rst Doc/library/concurrent.futures.rst -Doc/library/configparser.rst -Doc/library/contextlib.rst Doc/library/csv.rst Doc/library/datetime.rst Doc/library/dbm.rst @@ -56,26 +48,19 @@ Doc/library/exceptions.rst Doc/library/faulthandler.rst Doc/library/fcntl.rst Doc/library/ftplib.rst -Doc/library/functions.rst Doc/library/functools.rst -Doc/library/gettext.rst Doc/library/http.client.rst Doc/library/http.cookiejar.rst -Doc/library/http.cookies.rst Doc/library/http.server.rst Doc/library/importlib.rst -Doc/library/inspect.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst Doc/library/lzma.rst -Doc/library/mailbox.rst Doc/library/mmap.rst Doc/library/multiprocessing.rst Doc/library/multiprocessing.shared_memory.rst -Doc/library/numbers.rst Doc/library/optparse.rst -Doc/library/os.path.rst Doc/library/os.rst Doc/library/pickle.rst Doc/library/pickletools.rst @@ -85,13 +70,9 @@ Doc/library/profile.rst Doc/library/pyclbr.rst Doc/library/pydoc.rst Doc/library/pyexpat.rst -Doc/library/random.rst Doc/library/readline.rst -Doc/library/reprlib.rst Doc/library/resource.rst -Doc/library/rlcompleter.rst Doc/library/select.rst -Doc/library/shelve.rst Doc/library/signal.rst Doc/library/smtplib.rst Doc/library/socket.rst @@ -99,15 +80,11 @@ Doc/library/ssl.rst Doc/library/stdtypes.rst Doc/library/string.rst Doc/library/subprocess.rst -Doc/library/syslog.rst -Doc/library/tarfile.rst -Doc/library/tempfile.rst Doc/library/termios.rst Doc/library/test.rst Doc/library/tkinter.rst Doc/library/tkinter.scrolledtext.rst Doc/library/tkinter.ttk.rst -Doc/library/traceback.rst Doc/library/unittest.mock.rst Doc/library/unittest.rst Doc/library/urllib.parse.rst @@ -126,12 +103,10 @@ Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/reference/expressions.rst Doc/reference/import.rst -Doc/reference/simple_stmts.rst Doc/tutorial/datastructures.rst Doc/using/windows.rst Doc/whatsnew/2.1.rst Doc/whatsnew/2.2.rst -Doc/whatsnew/2.3.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 3551bfa4c0f133..ba37634545c2cf 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -126,7 +126,8 @@ def add_annotations(self, app, doctree): f"Object type mismatch in limited API annotation " f"for {name}: {record['role']!r} != {objtype!r}") stable_added = record['added'] - message = ' Part of the ' + message = sphinx_gettext('Part of the') + message = message.center(len(message) + 2) emph_node = nodes.emphasis(message, message, classes=['stableabi']) ref_node = addnodes.pending_xref( @@ -134,40 +135,40 @@ def add_annotations(self, app, doctree): reftype='ref', refexplicit="False") struct_abi_kind = record['struct_abi_kind'] if struct_abi_kind in {'opaque', 'members'}: - ref_node += nodes.Text('Limited API') + ref_node += nodes.Text(sphinx_gettext('Limited API')) else: - ref_node += nodes.Text('Stable ABI') + ref_node += nodes.Text(sphinx_gettext('Stable ABI')) emph_node += ref_node if struct_abi_kind == 'opaque': - emph_node += nodes.Text(' (as an opaque struct)') + emph_node += nodes.Text(' ' + sphinx_gettext('(as an opaque struct)')) elif struct_abi_kind == 'full-abi': - emph_node += nodes.Text(' (including all members)') + emph_node += nodes.Text(' ' + sphinx_gettext('(including all members)')) if record['ifdef_note']: emph_node += nodes.Text(' ' + record['ifdef_note']) if stable_added == '3.2': # Stable ABI was introduced in 3.2. pass else: - emph_node += nodes.Text(f' since version {stable_added}') + emph_node += nodes.Text(' ' + sphinx_gettext('since version %s') % stable_added) emph_node += nodes.Text('.') if struct_abi_kind == 'members': emph_node += nodes.Text( - ' (Only some members are part of the stable ABI.)') + ' ' + sphinx_gettext('(Only some members are part of the stable ABI.)')) node.insert(0, emph_node) # Unstable API annotation. if name.startswith('PyUnstable'): warn_node = nodes.admonition( classes=['unstable-c-api', 'warning']) - message = 'This is ' + message = sphinx_gettext('This is') + ' ' emph_node = nodes.emphasis(message, message) ref_node = addnodes.pending_xref( 'Unstable API', refdomain="std", reftarget='unstable-c-api', reftype='ref', refexplicit="False") - ref_node += nodes.Text('Unstable API') + ref_node += nodes.Text(sphinx_gettext('Unstable API')) emph_node += ref_node - emph_node += nodes.Text('. It may change without warning in minor releases.') + emph_node += nodes.Text(sphinx_gettext('. It may change without warning in minor releases.')) warn_node += emph_node node.insert(0, warn_node) diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index bab4aaeb4604b8..49c2a71a5e40cf 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -9,6 +9,16 @@ In extensions/c_annotations.py: +{% trans %}Part of the{% endtrans %} +{% trans %}Limited API{% endtrans %} +{% trans %}Stable ABI{% endtrans %} +{% trans %}(as an opaque struct){% endtrans %} +{% trans %}(including all members){% endtrans %} +{% trans %}since version %s{% endtrans %} +{% trans %}(Only some members are part of the stable ABI.){% endtrans %} +{% trans %}This is{% endtrans %} +{% trans %}Unstable API{% endtrans %} +{% trans %}. It may change without warning in minor releases.{% endtrans %} {% trans %}Return value: Always NULL.{% endtrans %} {% trans %}Return value: New reference.{% endtrans %} {% trans %}Return value: Borrowed reference.{% endtrans %} diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 7b92e1a51b6e67..3bf138ca225ee5 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -769,8 +769,10 @@ data from a string buffer instead, and pass it as an argument. or arithmetic operators, and assigning such a "pseudo-file" to sys.stdin will not cause the interpreter to read further input from it.) -Instance method objects have attributes, too: ``m.__self__`` is the instance -object with the method :meth:`!m`, and ``m.__func__`` is the function object +:ref:`Instance method objects ` have attributes, too: +:attr:`m.__self__ ` is the instance +object with the method :meth:`!m`, and :attr:`m.__func__ ` is +the :ref:`function object ` corresponding to the method. diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index aa9caa101da40a..77444f9cb8358d 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -559,10 +559,10 @@ defined to allow. For example:: def ask_ok(prompt, retries=4, reminder='Please try again!'): while True: - ok = input(prompt) - if ok in ('y', 'ye', 'yes'): + reply = input(prompt) + if reply in {'y', 'ye', 'yes'}: return True - if ok in ('n', 'no', 'nop', 'nope'): + if reply in {'n', 'no', 'nop', 'nope'}: return False retries = retries - 1 if retries < 0: diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index 30f3dfb6b238b4..0795e2fef98830 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -150,7 +150,7 @@ section. See `Examples of Floating Point Problems `_ for a pleasant summary of how binary floating-point works and the kinds of problems commonly encountered in practice. Also see -`The Perils of Floating Point `_ +`The Perils of Floating Point `_ for a more complete account of other common surprises. As that says near the end, "there are no easy answers." Still, don't be unduly diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index bf9e8e0b7b8066..0316239e776a95 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -437,7 +437,8 @@ When importing the package, Python searches through the directories on ``sys.path`` looking for the package subdirectory. The :file:`__init__.py` files are required to make Python treat directories -containing the file as packages. This prevents directories with a common name, +containing the file as packages (unless using a :term:`namespace package`, a +relatively advanced feature). This prevents directories with a common name, such as ``string``, from unintentionally hiding valid modules that occur later on the module search path. In the simplest case, :file:`__init__.py` can just be an empty file, but it can also execute initialization code for the package or diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 39c8d114f1e2c5..e032a1971bc6d6 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -590,9 +590,7 @@ Miscellaneous options .. versionadded:: 3.10 The ``-X warn_default_encoding`` option. - - .. deprecated-removed:: 3.9 3.10 - The ``-X oldparser`` option. + Removed the ``-X oldparser`` option. .. versionadded:: 3.11 The ``-X no_debug_ranges`` option. @@ -612,6 +610,29 @@ Miscellaneous options .. versionadded:: 3.13 The ``-X presite`` option. +.. _using-on-controlling-color: + +Controlling color +~~~~~~~~~~~~~~~~~ + +The Python interpreter is configured by default to use colors to highlight +output in certain situations such as when displaying tracebacks. This +behavior can be controlled by setting different environment variables. + +Setting the environment variable ``TERM`` to ``dumb`` will disable color. + +If the environment variable ``FORCE_COLOR`` is set, then color will be +enabled regardless of the value of TERM. This is useful on CI systems which +aren’t terminals but can none-the-less display ANSI escape sequences. + +If the environment variable ``NO_COLOR`` is set, Python will disable all color +in the output. This takes precedence over ``FORCE_COLOR``. + +All these environment variables are used also by other tools to control color +output. To control the color output only in the Python interpreter, the +:envvar:`PYTHON_COLORS` environment variable can be used. This variable takes +precedence over ``NO_COLOR``, which in turn takes precedence over +``FORCE_COLOR``. Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1110,6 +1131,13 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_COLORS + + If this variable is set to ``1``, the interpreter will colorize various kinds + of output. Setting it to ``0`` deactivates this behavior. + See also :ref:`using-on-controlling-color`. + + .. versionadded:: 3.13 Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index b51546e072a353..aab9469b44828a 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -287,10 +287,15 @@ General Options .. versionadded:: 3.11 +.. _free-threading-build: + .. option:: --disable-gil Enables **experimental** support for running Python without the - :term:`global interpreter lock` (GIL). + :term:`global interpreter lock` (GIL): free threading build. + + Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to + :data:`sys.abiflags`. See :pep:`703` "Making the Global Interpreter Lock Optional in CPython". @@ -740,6 +745,13 @@ Debug options .. versionadded:: 3.6 +.. option:: --with-thread-sanitizer + + Enable ThreadSanitizer data race detector, ``tsan`` + (default is no). + + .. versionadded:: 3.13 + Linker options -------------- diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index 6a5b3e294c6e85..50c88c4f43fc76 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -1030,7 +1030,7 @@ Module changes Lots of improvements and bugfixes were made to Python's extensive standard library; some of the affected modules include :mod:`readline`, -:mod:`!ConfigParser`, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, +:mod:`ConfigParser `, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, :mod:`!xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, and :mod:`!nntplib`. Consult the CVS logs for the exact patch-by-patch details. diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index f0e1ded75a9d27..6d2d3cc02b8768 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -424,7 +424,8 @@ PEP 232: Function Attributes In Python 2.1, functions can now have arbitrary information attached to them. People were often using docstrings to hold information about functions and -methods, because the ``__doc__`` attribute was the only way of attaching any +methods, because the :attr:`~function.__doc__` attribute was the only way of +attaching any information to a function. For example, in the Zope web application server, functions are marked as safe for public access by having a docstring, and in John Aycock's SPARK parsing framework, docstrings hold parts of the BNF grammar diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index af332b28a28231..8ebcbfaf248551 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -40,10 +40,10 @@ new feature. PEP 218: A Standard Set Datatype ================================ -The new :mod:`sets` module contains an implementation of a set datatype. The +The new :mod:`!sets` module contains an implementation of a set datatype. The :class:`Set` class is for mutable sets, sets that can have members added and -removed. The :class:`ImmutableSet` class is for sets that can't be modified, -and instances of :class:`ImmutableSet` can therefore be used as dictionary keys. +removed. The :class:`!ImmutableSet` class is for sets that can't be modified, +and instances of :class:`!ImmutableSet` can therefore be used as dictionary keys. Sets are built on top of dictionaries, so the elements within a set must be hashable. @@ -63,10 +63,10 @@ Here's a simple example:: Set([1, 2, 5]) >>> -The union and intersection of sets can be computed with the :meth:`union` and -:meth:`intersection` methods; an alternative notation uses the bitwise operators +The union and intersection of sets can be computed with the :meth:`~frozenset.union` and +:meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`union_update` and :meth:`intersection_update`. :: +:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) @@ -97,7 +97,7 @@ in-place version with the ungainly name :meth:`symmetric_difference_update`. :: Set([1, 2, 5, 6]) >>> -There are also :meth:`issubset` and :meth:`issuperset` methods for checking +There are also :meth:`!issubset` and :meth:`!issuperset` methods for checking whether one set is a subset or superset of another:: >>> S1 = sets.Set([1,2,3]) @@ -166,7 +166,7 @@ statement isn't allowed inside the :keyword:`try` block of a :keyword:`!try`...\ :keyword:`!finally` statement; read :pep:`255` for a full explanation of the interaction between :keyword:`!yield` and exceptions.) -Here's a sample usage of the :func:`generate_ints` generator:: +Here's a sample usage of the :func:`!generate_ints` generator:: >>> gen = generate_ints(3) >>> gen @@ -227,7 +227,7 @@ like:: sentence := "Store it in the neighboring harbor" if (i := find("or", sentence)) > 5 then write(i) -In Icon the :func:`find` function returns the indexes at which the substring +In Icon the :func:`!find` function returns the indexes at which the substring "or" is found: 3, 23, 33. In the :keyword:`if` statement, ``i`` is first assigned a value of 3, but 3 is less than 5, so the comparison fails, and Icon retries it with the second value of 23. 23 is greater than 5, so the comparison @@ -345,7 +345,7 @@ Python now allows using arbitrary Unicode strings (within the limitations of the file system) for all functions that expect file names, most notably the :func:`open` built-in function. If a Unicode string is passed to :func:`os.listdir`, Python now returns a list of Unicode strings. A new -function, :func:`os.getcwdu`, returns the current directory as a Unicode string. +function, :func:`!os.getcwdu`, returns the current directory as a Unicode string. Byte strings still work as file names, and on Windows Python will transparently convert them to Unicode using the ``mbcs`` encoding. @@ -386,10 +386,10 @@ one followed by the platform on which Python is running. Opening a file with the mode ``'U'`` or ``'rU'`` will open a file for reading in :term:`universal newlines` mode. All three line ending conventions will be translated to a ``'\n'`` in the strings returned by the various file methods such as -:meth:`read` and :meth:`readline`. +:meth:`!read` and :meth:`!readline`. Universal newline support is also used when importing modules and when executing -a file with the :func:`execfile` function. This means that Python modules can +a file with the :func:`!execfile` function. This means that Python modules can be shared between all three operating systems without needing to convert the line-endings. @@ -450,16 +450,16 @@ records to standard error or to a file or socket, send them to the system log, or even e-mail them to a particular address; of course, it's also possible to write your own handler classes. -The :class:`Logger` class is the primary class. Most application code will deal -with one or more :class:`Logger` objects, each one used by a particular -subsystem of the application. Each :class:`Logger` is identified by a name, and +The :class:`~logging.Logger` class is the primary class. Most application code will deal +with one or more :class:`~logging.Logger` objects, each one used by a particular +subsystem of the application. Each :class:`~logging.Logger` is identified by a name, and names are organized into a hierarchy using ``.`` as the component separator. -For example, you might have :class:`Logger` instances named ``server``, +For example, you might have :class:`~logging.Logger` instances named ``server``, ``server.auth`` and ``server.network``. The latter two instances are below ``server`` in the hierarchy. This means that if you turn up the verbosity for ``server`` or direct ``server`` messages to a different handler, the changes will also apply to records logged to ``server.auth`` and ``server.network``. -There's also a root :class:`Logger` that's the parent of all other loggers. +There's also a root :class:`~logging.Logger` that's the parent of all other loggers. For simple uses, the :mod:`logging` package contains some convenience functions that always use the root log:: @@ -480,14 +480,14 @@ This produces the following output:: In the default configuration, informational and debugging messages are suppressed and the output is sent to standard error. You can enable the display -of informational and debugging messages by calling the :meth:`setLevel` method +of informational and debugging messages by calling the :meth:`~logging.Logger.setLevel` method on the root logger. -Notice the :func:`warning` call's use of string formatting operators; all of the +Notice the :func:`~logging.warning` call's use of string formatting operators; all of the functions for logging messages take the arguments ``(msg, arg1, arg2, ...)`` and log the string resulting from ``msg % (arg1, arg2, ...)``. -There's also an :func:`exception` function that records the most recent +There's also an :func:`~logging.exception` function that records the most recent traceback. Any of the other functions will also record the traceback if you specify a true value for the keyword argument *exc_info*. :: @@ -517,16 +517,16 @@ it if it doesn't exist yet. ``getLogger(None)`` returns the root logger. :: ... Log records are usually propagated up the hierarchy, so a message logged to -``server.auth`` is also seen by ``server`` and ``root``, but a :class:`Logger` -can prevent this by setting its :attr:`propagate` attribute to :const:`False`. +``server.auth`` is also seen by ``server`` and ``root``, but a :class:`~logging.Logger` +can prevent this by setting its :attr:`~logging.Logger.propagate` attribute to :const:`False`. There are more classes provided by the :mod:`logging` package that can be -customized. When a :class:`Logger` instance is told to log a message, it -creates a :class:`LogRecord` instance that is sent to any number of different -:class:`Handler` instances. Loggers and handlers can also have an attached list -of filters, and each filter can cause the :class:`LogRecord` to be ignored or +customized. When a :class:`~logging.Logger` instance is told to log a message, it +creates a :class:`~logging.LogRecord` instance that is sent to any number of different +:class:`~logging.Handler` instances. Loggers and handlers can also have an attached list +of filters, and each filter can cause the :class:`~logging.LogRecord` to be ignored or can modify the record before passing it along. When they're finally output, -:class:`LogRecord` instances are converted to text by a :class:`Formatter` +:class:`~logging.LogRecord` instances are converted to text by a :class:`~logging.Formatter` class. All of these classes can be replaced by your own specially written classes. @@ -550,7 +550,7 @@ PEP 285: A Boolean Type ======================= A Boolean type was added to Python 2.3. Two new constants were added to the -:mod:`__builtin__` module, :const:`True` and :const:`False`. (:const:`True` and +:mod:`!__builtin__` module, :const:`True` and :const:`False`. (:const:`True` and :const:`False` constants were added to the built-ins in Python 2.2.1, but the 2.2.1 versions are simply set to integer values of 1 and 0 and aren't a different type.) @@ -662,7 +662,7 @@ a central catalog server. The resulting catalog is available from https://pypi.org. To make the catalog a bit more useful, a new optional *classifiers* keyword -argument has been added to the Distutils :func:`setup` function. A list of +argument has been added to the Distutils :func:`!setup` function. A list of `Trove `_-style strings can be supplied to help classify the software. @@ -703,14 +703,14 @@ PEP 302: New Import Hooks ========================= While it's been possible to write custom import hooks ever since the -:mod:`ihooks` module was introduced in Python 1.3, no one has ever been really +:mod:`!ihooks` module was introduced in Python 1.3, no one has ever been really happy with it because writing new import hooks is difficult and messy. There -have been various proposed alternatives such as the :mod:`imputil` and :mod:`iu` +have been various proposed alternatives such as the :mod:`!imputil` and :mod:`!iu` modules, but none of them has ever gained much acceptance, and none of them were easily usable from C code. :pep:`302` borrows ideas from its predecessors, especially from Gordon -McMillan's :mod:`iu` module. Three new items are added to the :mod:`sys` +McMillan's :mod:`!iu` module. Three new items are added to the :mod:`sys` module: * ``sys.path_hooks`` is a list of callable objects; most often they'll be @@ -790,7 +790,7 @@ package is much simpler:: for line in reader: print line -The :func:`reader` function takes a number of different options. The field +The :func:`~csv.reader` function takes a number of different options. The field separator isn't limited to the comma and can be changed to any character, and so can the quoting and line-ending characters. @@ -814,7 +814,7 @@ of tuples or lists, quoting strings that contain the delimiter. PEP 307: Pickle Enhancements ============================ -The :mod:`pickle` and :mod:`cPickle` modules received some attention during the +The :mod:`pickle` and :mod:`!cPickle` modules received some attention during the 2.3 development cycle. In 2.2, new-style classes could be pickled without difficulty, but they weren't pickled very compactly; :pep:`307` quotes a trivial example where a new-style class results in a pickled string three times longer @@ -829,13 +829,13 @@ fanciest protocol available. Unpickling is no longer considered a safe operation. 2.2's :mod:`pickle` provided hooks for trying to prevent unsafe classes from being unpickled -(specifically, a :attr:`__safe_for_unpickling__` attribute), but none of this +(specifically, a :attr:`!__safe_for_unpickling__` attribute), but none of this code was ever audited and therefore it's all been ripped out in 2.3. You should not unpickle untrusted data in any version of Python. To reduce the pickling overhead for new-style classes, a new interface for customizing pickling was added using three special methods: -:meth:`__getstate__`, :meth:`__setstate__`, and :meth:`__getnewargs__`. Consult +:meth:`~object.__getstate__`, :meth:`~object.__setstate__`, and :meth:`~object.__getnewargs__`. Consult :pep:`307` for the full semantics of these methods. As a way to compress pickles yet further, it's now possible to use integer codes @@ -939,7 +939,7 @@ Or use slice objects directly in subscripts:: To simplify implementing sequences that support extended slicing, slice objects now have a method ``indices(length)`` which, given the length of a sequence, returns a ``(start, stop, step)`` tuple that can be passed directly to -:func:`range`. :meth:`indices` handles omitted and out-of-bounds indices in a +:func:`range`. :meth:`!indices` handles omitted and out-of-bounds indices in a manner consistent with regular slices (and this innocuous phrase hides a welter of confusing details!). The method is intended to be used like this:: @@ -1042,7 +1042,7 @@ Here are all of the changes that Python 2.3 makes to the core Python language. execute any assertions. * Most type objects are now callable, so you can use them to create new objects - such as functions, classes, and modules. (This means that the :mod:`new` module + such as functions, classes, and modules. (This means that the :mod:`!new` module can be deprecated in a future Python version, because you can now use the type objects available in the :mod:`types` module.) For example, you can create a new module object with the following code: @@ -1069,11 +1069,11 @@ Here are all of the changes that Python 2.3 makes to the core Python language. * Using ``None`` as a variable name will now result in a :exc:`SyntaxWarning` warning. In a future version of Python, ``None`` may finally become a keyword. -* The :meth:`xreadlines` method of file objects, introduced in Python 2.1, is no +* The :meth:`!xreadlines` method of file objects, introduced in Python 2.1, is no longer necessary because files now behave as their own iterator. - :meth:`xreadlines` was originally introduced as a faster way to loop over all + :meth:`!xreadlines` was originally introduced as a faster way to loop over all the lines in a file, but now you can simply write ``for line in file_obj``. - File objects also have a new read-only :attr:`encoding` attribute that gives the + File objects also have a new read-only :attr:`!encoding` attribute that gives the encoding used by the file; Unicode strings written to the file will be automatically converted to bytes using the given encoding. @@ -1096,12 +1096,12 @@ Here are all of the changes that Python 2.3 makes to the core Python language. switching overhead. Some multithreaded applications may suffer slower response time, but that's easily fixed by setting the limit back to a lower number using ``sys.setcheckinterval(N)``. The limit can be retrieved with the new - :func:`sys.getcheckinterval` function. + :func:`!sys.getcheckinterval` function. * One minor but far-reaching change is that the names of extension types defined by the modules included with Python now contain the module and a ``'.'`` in front of the type name. For example, in Python 2.2, if you created a socket and - printed its :attr:`__class__`, you'd get this output:: + printed its :attr:`!__class__`, you'd get this output:: >>> s = socket.socket() >>> s.__class__ @@ -1138,9 +1138,9 @@ String Changes True Note that this doesn't tell you where the substring starts; if you need that - information, use the :meth:`find` string method. + information, use the :meth:`~str.find` string method. -* The :meth:`strip`, :meth:`lstrip`, and :meth:`rstrip` string methods now have +* The :meth:`~str.strip`, :meth:`~str.lstrip`, and :meth:`~str.rstrip` string methods now have an optional argument for specifying the characters to strip. The default is still to remove all whitespace characters:: @@ -1156,13 +1156,13 @@ String Changes (Suggested by Simon Brunning and implemented by Walter Dörwald.) -* The :meth:`startswith` and :meth:`endswith` string methods now accept negative +* The :meth:`~str.startswith` and :meth:`~str.endswith` string methods now accept negative numbers for the *start* and *end* parameters. -* Another new string method is :meth:`zfill`, originally a function in the - :mod:`string` module. :meth:`zfill` pads a numeric string with zeros on the +* Another new string method is :meth:`~str.zfill`, originally a function in the + :mod:`string` module. :meth:`~str.zfill` pads a numeric string with zeros on the left until it's the specified width. Note that the ``%`` operator is still more - flexible and powerful than :meth:`zfill`. :: + flexible and powerful than :meth:`~str.zfill`. :: >>> '45'.zfill(4) '0045' @@ -1173,10 +1173,10 @@ String Changes (Contributed by Walter Dörwald.) -* A new type object, :class:`basestring`, has been added. Both 8-bit strings and +* A new type object, :class:`!basestring`, has been added. Both 8-bit strings and Unicode strings inherit from this type, so ``isinstance(obj, basestring)`` will return :const:`True` for either kind of string. It's a completely abstract - type, so you can't create :class:`basestring` instances. + type, so you can't create :class:`!basestring` instances. * Interned strings are no longer immortal and will now be garbage-collected in the usual way when the only reference to them is from the internal dictionary of @@ -1191,7 +1191,7 @@ Optimizations * The creation of new-style class instances has been made much faster; they're now faster than classic classes! -* The :meth:`sort` method of list objects has been extensively rewritten by Tim +* The :meth:`~list.sort` method of list objects has been extensively rewritten by Tim Peters, and the implementation is significantly faster. * Multiplication of large long integers is now much faster thanks to an @@ -1203,7 +1203,7 @@ Optimizations increase, depending on your compiler's idiosyncrasies. See section :ref:`23section-other` for a longer explanation. (Removed by Michael Hudson.) -* :func:`xrange` objects now have their own iterator, making ``for i in +* :func:`!xrange` objects now have their own iterator, making ``for i in xrange(n)`` slightly faster than ``for i in range(n)``. (Patch by Raymond Hettinger.) @@ -1230,21 +1230,21 @@ complete list of changes, or look through the CVS logs for all the details. operator to add another array's contents, and the ``*=`` assignment operator to repeat an array. (Contributed by Jason Orendorff.) -* The :mod:`bsddb` module has been replaced by version 4.1.6 of the `PyBSDDB +* The :mod:`!bsddb` module has been replaced by version 4.1.6 of the `PyBSDDB `_ package, providing a more complete interface to the transactional features of the BerkeleyDB library. - The old version of the module has been renamed to :mod:`bsddb185` and is no + The old version of the module has been renamed to :mod:`!bsddb185` and is no longer built automatically; you'll have to edit :file:`Modules/Setup` to enable - it. Note that the new :mod:`bsddb` package is intended to be compatible with + it. Note that the new :mod:`!bsddb` package is intended to be compatible with the old module, so be sure to file bugs if you discover any incompatibilities. When upgrading to Python 2.3, if the new interpreter is compiled with a new version of the underlying BerkeleyDB library, you will almost certainly have to convert your database files to the new version. You can do this fairly easily with the new scripts :file:`db2pickle.py` and :file:`pickle2db.py` which you will find in the distribution's :file:`Tools/scripts` directory. If you've - already been using the PyBSDDB package and importing it as :mod:`bsddb3`, you - will have to change your ``import`` statements to import it as :mod:`bsddb`. + already been using the PyBSDDB package and importing it as :mod:`!bsddb3`, you + will have to change your ``import`` statements to import it as :mod:`!bsddb`. * The new :mod:`bz2` module is an interface to the bz2 data compression library. bz2-compressed data is usually smaller than corresponding @@ -1253,11 +1253,11 @@ complete list of changes, or look through the CVS logs for all the details. * A set of standard date/time types has been added in the new :mod:`datetime` module. See the following section for more details. -* The Distutils :class:`Extension` class now supports an extra constructor +* The Distutils :class:`!Extension` class now supports an extra constructor argument named *depends* for listing additional source files that an extension depends on. This lets Distutils recompile the module if any of the dependency files are modified. For example, if :file:`sampmodule.c` includes the header - file :file:`sample.h`, you would create the :class:`Extension` object like + file :file:`sample.h`, you would create the :class:`!Extension` object like this:: ext = Extension("samp", @@ -1268,21 +1268,21 @@ complete list of changes, or look through the CVS logs for all the details. (Contributed by Jeremy Hylton.) * Other minor changes to Distutils: it now checks for the :envvar:`CC`, - :envvar:`CFLAGS`, :envvar:`CPP`, :envvar:`LDFLAGS`, and :envvar:`CPPFLAGS` + :envvar:`CFLAGS`, :envvar:`!CPP`, :envvar:`LDFLAGS`, and :envvar:`CPPFLAGS` environment variables, using them to override the settings in Python's configuration (contributed by Robert Weber). * Previously the :mod:`doctest` module would only search the docstrings of public methods and functions for test cases, but it now also examines private - ones as well. The :func:`DocTestSuite` function creates a + ones as well. The :func:`~doctest.DocTestSuite` function creates a :class:`unittest.TestSuite` object from a set of :mod:`doctest` tests. * The new ``gc.get_referents(object)`` function returns a list of all the objects referenced by *object*. -* The :mod:`getopt` module gained a new function, :func:`gnu_getopt`, that - supports the same arguments as the existing :func:`getopt` function but uses - GNU-style scanning mode. The existing :func:`getopt` stops processing options as +* The :mod:`getopt` module gained a new function, :func:`~getopt.gnu_getopt`, that + supports the same arguments as the existing :func:`~getopt.getopt` function but uses + GNU-style scanning mode. The existing :func:`~getopt.getopt` stops processing options as soon as a non-option argument is encountered, but in GNU-style mode processing continues, meaning that options and arguments can be mixed. For example:: @@ -1311,7 +1311,7 @@ complete list of changes, or look through the CVS logs for all the details. O(lg n). (See https://xlinux.nist.gov/dads//HTML/priorityque.html for more information about the priority queue data structure.) - The :mod:`heapq` module provides :func:`heappush` and :func:`heappop` functions + The :mod:`heapq` module provides :func:`~heapq.heappush` and :func:`~heapq.heappop` functions for adding and removing items while maintaining the heap property on top of some other mutable Python sequence type. Here's an example that uses a Python list:: @@ -1343,7 +1343,7 @@ complete list of changes, or look through the CVS logs for all the details. * The :mod:`itertools` contains a number of useful functions for use with iterators, inspired by various functions provided by the ML and Haskell languages. For example, ``itertools.ifilter(predicate, iterator)`` returns all - elements in the iterator for which the function :func:`predicate` returns + elements in the iterator for which the function :func:`!predicate` returns :const:`True`, and ``itertools.repeat(obj, N)`` returns ``obj`` *N* times. There are a number of other functions in the module; see the package's reference documentation for details. @@ -1356,21 +1356,21 @@ complete list of changes, or look through the CVS logs for all the details. was added to :func:`math.log` to make it easier to compute logarithms for bases other than ``e`` and ``10``. (Contributed by Raymond Hettinger.) -* Several new POSIX functions (:func:`getpgid`, :func:`killpg`, :func:`lchown`, - :func:`loadavg`, :func:`major`, :func:`makedev`, :func:`minor`, and - :func:`mknod`) were added to the :mod:`posix` module that underlies the +* Several new POSIX functions (:func:`!getpgid`, :func:`!killpg`, :func:`!lchown`, + :func:`!loadavg`, :func:`!major`, :func:`!makedev`, :func:`!minor`, and + :func:`!mknod`) were added to the :mod:`posix` module that underlies the :mod:`os` module. (Contributed by Gustavo Niemeyer, Geert Jansen, and Denis S. Otkidach.) -* In the :mod:`os` module, the :func:`\*stat` family of functions can now report +* In the :mod:`os` module, the :func:`!\*stat` family of functions can now report fractions of a second in a timestamp. Such time stamps are represented as floats, similar to the value returned by :func:`time.time`. During testing, it was found that some applications will break if time stamps are floats. For compatibility, when using the tuple interface of the - :class:`stat_result` time stamps will be represented as integers. When using + :class:`~os.stat_result` time stamps will be represented as integers. When using named fields (a feature first introduced in Python 2.2), time stamps are still - represented as integers, unless :func:`os.stat_float_times` is invoked to enable + represented as integers, unless :func:`!os.stat_float_times` is invoked to enable float return values:: >>> os.stat("/tmp").st_mtime @@ -1391,7 +1391,7 @@ complete list of changes, or look through the CVS logs for all the details. automatically generate a usage message. See the following section for more details. -* The old and never-documented :mod:`linuxaudiodev` module has been deprecated, +* The old and never-documented :mod:`!linuxaudiodev` module has been deprecated, and a new version named :mod:`!ossaudiodev` has been added. The module was renamed because the OSS sound drivers can be used on platforms other than Linux, and the interface has also been tidied and brought up to date in various ways. @@ -1402,14 +1402,14 @@ complete list of changes, or look through the CVS logs for all the details. functions for getting the architecture, CPU type, the Windows OS version, and even the Linux distribution version. (Contributed by Marc-André Lemburg.) -* The parser objects provided by the :mod:`pyexpat` module can now optionally +* The parser objects provided by the :mod:`pyexpat ` module can now optionally buffer character data, resulting in fewer calls to your character data handler and therefore faster performance. Setting the parser object's - :attr:`buffer_text` attribute to :const:`True` will enable buffering. + :attr:`~xml.parsers.expat.xmlparser.buffer_text` attribute to :const:`True` will enable buffering. * The ``sample(population, k)`` function was added to the :mod:`random` - module. *population* is a sequence or :class:`xrange` object containing the - elements of a population, and :func:`sample` chooses *k* elements from the + module. *population* is a sequence or :class:`!xrange` object containing the + elements of a population, and :func:`~random.sample` chooses *k* elements from the population without replacing chosen elements. *k* can be any value up to ``len(population)``. For example:: @@ -1436,20 +1436,20 @@ complete list of changes, or look through the CVS logs for all the details. (All changes contributed by Raymond Hettinger.) * The :mod:`readline` module also gained a number of new functions: - :func:`get_history_item`, :func:`get_current_history_length`, and - :func:`redisplay`. + :func:`~readline.get_history_item`, :func:`~readline.get_current_history_length`, and + :func:`~readline.redisplay`. -* The :mod:`rexec` and :mod:`Bastion` modules have been declared dead, and +* The :mod:`!rexec` and :mod:`!Bastion` modules have been declared dead, and attempts to import them will fail with a :exc:`RuntimeError`. New-style classes provide new ways to break out of the restricted execution environment provided - by :mod:`rexec`, and no one has interest in fixing them or time to do so. If - you have applications using :mod:`rexec`, rewrite them to use something else. + by :mod:`!rexec`, and no one has interest in fixing them or time to do so. If + you have applications using :mod:`!rexec`, rewrite them to use something else. (Sticking with Python 2.2 or 2.1 will not make your applications any safer - because there are known bugs in the :mod:`rexec` module in those versions. To - repeat: if you're using :mod:`rexec`, stop using it immediately.) + because there are known bugs in the :mod:`!rexec` module in those versions. To + repeat: if you're using :mod:`!rexec`, stop using it immediately.) -* The :mod:`rotor` module has been deprecated because the algorithm it uses for +* The :mod:`!rotor` module has been deprecated because the algorithm it uses for encryption is not believed to be secure. If you need encryption, use one of the several AES Python modules that are available separately. @@ -1474,9 +1474,9 @@ complete list of changes, or look through the CVS logs for all the details. * On Windows, the :mod:`socket` module now ships with Secure Sockets Layer (SSL) support. -* The value of the C :c:macro:`PYTHON_API_VERSION` macro is now exposed at the +* The value of the C :c:macro:`!PYTHON_API_VERSION` macro is now exposed at the Python level as ``sys.api_version``. The current exception can be cleared by - calling the new :func:`sys.exc_clear` function. + calling the new :func:`!sys.exc_clear` function. * The new :mod:`tarfile` module allows reading from and writing to :program:`tar`\ -format archive files. (Contributed by Lars Gustäbel.) @@ -1486,7 +1486,7 @@ complete list of changes, or look through the CVS logs for all the details. string and returns a list containing the text split into lines of no more than the chosen width. The ``fill(text, width)`` function returns a single string, reformatted to fit into lines no longer than the chosen width. (As you - can guess, :func:`fill` is built on top of :func:`wrap`. For example:: + can guess, :func:`~textwrap.fill` is built on top of :func:`~textwrap.wrap`. For example:: >>> import textwrap >>> paragraph = "Not a whit, we defy augury: ... more text ..." @@ -1503,15 +1503,15 @@ complete list of changes, or look through the CVS logs for all the details. it will come: the readiness is all. >>> - The module also contains a :class:`TextWrapper` class that actually implements - the text wrapping strategy. Both the :class:`TextWrapper` class and the - :func:`wrap` and :func:`fill` functions support a number of additional keyword + The module also contains a :class:`~textwrap.TextWrapper` class that actually implements + the text wrapping strategy. Both the :class:`~textwrap.TextWrapper` class and the + :func:`~textwrap.wrap` and :func:`~textwrap.fill` functions support a number of additional keyword arguments for fine-tuning the formatting; consult the module's documentation for details. (Contributed by Greg Ward.) -* The :mod:`thread` and :mod:`threading` modules now have companion modules, - :mod:`dummy_thread` and :mod:`dummy_threading`, that provide a do-nothing - implementation of the :mod:`thread` module's interface for platforms where +* The :mod:`!thread` and :mod:`threading` modules now have companion modules, + :mod:`!dummy_thread` and :mod:`!dummy_threading`, that provide a do-nothing + implementation of the :mod:`!thread` module's interface for platforms where threads are not supported. The intention is to simplify thread-aware modules (ones that *don't* rely on threads to run) by putting the following code at the top:: @@ -1521,26 +1521,26 @@ complete list of changes, or look through the CVS logs for all the details. except ImportError: import dummy_threading as _threading - In this example, :mod:`_threading` is used as the module name to make it clear + In this example, :mod:`!_threading` is used as the module name to make it clear that the module being used is not necessarily the actual :mod:`threading` - module. Code can call functions and use classes in :mod:`_threading` whether or + module. Code can call functions and use classes in :mod:`!_threading` whether or not threads are supported, avoiding an :keyword:`if` statement and making the code slightly clearer. This module will not magically make multithreaded code run without threads; code that waits for another thread to return or to do something will simply hang forever. -* The :mod:`time` module's :func:`strptime` function has long been an annoyance - because it uses the platform C library's :func:`strptime` implementation, and +* The :mod:`time` module's :func:`~time.strptime` function has long been an annoyance + because it uses the platform C library's :func:`~time.strptime` implementation, and different platforms sometimes have odd bugs. Brett Cannon contributed a portable implementation that's written in pure Python and should behave identically on all platforms. * The new :mod:`timeit` module helps measure how long snippets of Python code take to execute. The :file:`timeit.py` file can be run directly from the - command line, or the module's :class:`Timer` class can be imported and used + command line, or the module's :class:`~timeit.Timer` class can be imported and used directly. Here's a short example that figures out whether it's faster to convert an 8-bit string to Unicode by appending an empty Unicode string to it or - by using the :func:`unicode` function:: + by using the :func:`!unicode` function:: import timeit @@ -1558,46 +1558,46 @@ complete list of changes, or look through the CVS logs for all the details. * The :mod:`!Tix` module has received various bug fixes and updates for the current version of the Tix package. -* The :mod:`Tkinter` module now works with a thread-enabled version of Tcl. +* The :mod:`!Tkinter` module now works with a thread-enabled version of Tcl. Tcl's threading model requires that widgets only be accessed from the thread in which they're created; accesses from another thread can cause Tcl to panic. For - certain Tcl interfaces, :mod:`Tkinter` will now automatically avoid this when a + certain Tcl interfaces, :mod:`!Tkinter` will now automatically avoid this when a widget is accessed from a different thread by marshalling a command, passing it to the correct thread, and waiting for the results. Other interfaces can't be - handled automatically but :mod:`Tkinter` will now raise an exception on such an + handled automatically but :mod:`!Tkinter` will now raise an exception on such an access so that you can at least find out about the problem. See https://mail.python.org/pipermail/python-dev/2002-December/031107.html for a more detailed explanation of this change. (Implemented by Martin von Löwis.) -* Calling Tcl methods through :mod:`_tkinter` no longer returns only strings. +* Calling Tcl methods through :mod:`!_tkinter` no longer returns only strings. Instead, if Tcl returns other objects those objects are converted to their - Python equivalent, if one exists, or wrapped with a :class:`_tkinter.Tcl_Obj` + Python equivalent, if one exists, or wrapped with a :class:`!_tkinter.Tcl_Obj` object if no Python equivalent exists. This behavior can be controlled through - the :meth:`wantobjects` method of :class:`tkapp` objects. + the :meth:`!wantobjects` method of :class:`!tkapp` objects. - When using :mod:`_tkinter` through the :mod:`Tkinter` module (as most Tkinter + When using :mod:`!_tkinter` through the :mod:`!Tkinter` module (as most Tkinter applications will), this feature is always activated. It should not cause compatibility problems, since Tkinter would always convert string results to Python types where possible. If any incompatibilities are found, the old behavior can be restored by setting - the :attr:`wantobjects` variable in the :mod:`Tkinter` module to false before - creating the first :class:`tkapp` object. :: + the :attr:`!wantobjects` variable in the :mod:`!Tkinter` module to false before + creating the first :class:`!tkapp` object. :: import Tkinter Tkinter.wantobjects = 0 Any breakage caused by this change should be reported as a bug. -* The :mod:`UserDict` module has a new :class:`DictMixin` class which defines +* The :mod:`!UserDict` module has a new :class:`!DictMixin` class which defines all dictionary methods for classes that already have a minimum mapping interface. This greatly simplifies writing classes that need to be substitutable for dictionaries, such as the classes in the :mod:`shelve` module. Adding the mix-in as a superclass provides the full dictionary interface - whenever the class defines :meth:`~object.__getitem__`, :meth:`__setitem__`, - :meth:`__delitem__`, and :meth:`keys`. For example:: + whenever the class defines :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, + :meth:`~object.__delitem__`, and :meth:`!keys`. For example:: >>> import UserDict >>> class SeqDict(UserDict.DictMixin): @@ -1640,15 +1640,15 @@ complete list of changes, or look through the CVS logs for all the details. * The DOM implementation in :mod:`xml.dom.minidom` can now generate XML output in a particular encoding by providing an optional encoding argument to the - :meth:`toxml` and :meth:`toprettyxml` methods of DOM nodes. + :meth:`~xml.dom.minidom.Node.toxml` and :meth:`~xml.dom.minidom.Node.toprettyxml` methods of DOM nodes. -* The :mod:`xmlrpclib` module now supports an XML-RPC extension for handling nil +* The :mod:`!xmlrpclib` module now supports an XML-RPC extension for handling nil data values such as Python's ``None``. Nil values are always supported on unmarshalling an XML-RPC response. To generate requests containing ``None``, you must supply a true value for the *allow_none* parameter when creating a - :class:`Marshaller` instance. + :class:`!Marshaller` instance. -* The new :mod:`DocXMLRPCServer` module allows writing self-documenting XML-RPC +* The new :mod:`!DocXMLRPCServer` module allows writing self-documenting XML-RPC servers. Run it in demo mode (as a program) to see it in action. Pointing the web browser to the RPC server produces pydoc-style documentation; pointing xmlrpclib to the server allows invoking the actual methods. (Contributed by @@ -1663,8 +1663,8 @@ complete list of changes, or look through the CVS logs for all the details. The :mod:`socket` module has also been extended to transparently convert Unicode hostnames to the ACE version before passing them to the C library. - Modules that deal with hostnames such as :mod:`httplib` and :mod:`ftplib`) - also support Unicode host names; :mod:`httplib` also sends HTTP ``Host`` + Modules that deal with hostnames such as :mod:`!httplib` and :mod:`ftplib`) + also support Unicode host names; :mod:`!httplib` also sends HTTP ``Host`` headers using the ACE version of the domain name. :mod:`urllib` supports Unicode URLs with non-ASCII host names as long as the ``path`` part of the URL is ASCII only. @@ -1682,17 +1682,17 @@ Date and time types suitable for expressing timestamps were added as the :mod:`datetime` module. The types don't support different calendars or many fancy features, and just stick to the basics of representing time. -The three primary types are: :class:`date`, representing a day, month, and year; +The three primary types are: :class:`~datetime.date`, representing a day, month, and year; :class:`~datetime.time`, consisting of hour, minute, and second; and :class:`~datetime.datetime`, -which contains all the attributes of both :class:`date` and :class:`~datetime.time`. -There's also a :class:`timedelta` class representing differences between two +which contains all the attributes of both :class:`~datetime.date` and :class:`~datetime.time`. +There's also a :class:`~datetime.timedelta` class representing differences between two points in time, and time zone logic is implemented by classes inheriting from -the abstract :class:`tzinfo` class. +the abstract :class:`~datetime.tzinfo` class. -You can create instances of :class:`date` and :class:`~datetime.time` by either supplying +You can create instances of :class:`~datetime.date` and :class:`~datetime.time` by either supplying keyword arguments to the appropriate constructor, e.g. ``datetime.date(year=1972, month=10, day=15)``, or by using one of a number of -class methods. For example, the :meth:`date.today` class method returns the +class methods. For example, the :meth:`~datetime.date.today` class method returns the current local date. Once created, instances of the date/time classes are all immutable. There are a @@ -1707,8 +1707,8 @@ number of methods for producing formatted strings from objects:: >>> now.strftime('%Y %d %b') '2002 30 Dec' -The :meth:`replace` method allows modifying one or more fields of a -:class:`date` or :class:`~datetime.datetime` instance, returning a new instance:: +The :meth:`~datetime.datetime.replace` method allows modifying one or more fields of a +:class:`~datetime.date` or :class:`~datetime.datetime` instance, returning a new instance:: >>> d = datetime.datetime.now() >>> d @@ -1718,10 +1718,10 @@ The :meth:`replace` method allows modifying one or more fields of a >>> Instances can be compared, hashed, and converted to strings (the result is the -same as that of :meth:`isoformat`). :class:`date` and :class:`~datetime.datetime` -instances can be subtracted from each other, and added to :class:`timedelta` +same as that of :meth:`~datetime.datetime.isoformat`). :class:`~datetime.date` and :class:`~datetime.datetime` +instances can be subtracted from each other, and added to :class:`~datetime.timedelta` instances. The largest missing feature is that there's no standard library -support for parsing strings and getting back a :class:`date` or +support for parsing strings and getting back a :class:`~datetime.date` or :class:`~datetime.datetime`. For more information, refer to the module's reference documentation. @@ -1739,7 +1739,7 @@ command-line parsing that follows the Unix conventions, automatically creates the output for :option:`!--help`, and can perform different actions for different options. -You start by creating an instance of :class:`OptionParser` and telling it what +You start by creating an instance of :class:`~optparse.OptionParser` and telling it what your program's options are. :: import sys @@ -1753,7 +1753,7 @@ your program's options are. :: action='store', type='int', dest='length', help='set maximum length of output') -Parsing a command line is then done by calling the :meth:`parse_args` method. :: +Parsing a command line is then done by calling the :meth:`~optparse.OptionParser.parse_args` method. :: options, args = op.parse_args(sys.argv[1:]) print options @@ -1925,7 +1925,7 @@ Changes to Python's build process and to the C API include: dependence on a system version or local installation of Expat. * If you dynamically allocate type objects in your extension, you should be - aware of a change in the rules relating to the :attr:`__module__` and + aware of a change in the rules relating to the :attr:`!__module__` and :attr:`~definition.__name__` attributes. In summary, you will want to ensure the type's dictionary contains a ``'__module__'`` key; making the module name the part of the type name leading up to the final period will no longer have the desired @@ -1940,7 +1940,7 @@ Port-Specific Changes Support for a port to IBM's OS/2 using the EMX runtime environment was merged into the main Python source tree. EMX is a POSIX emulation layer over the OS/2 system APIs. The Python port for EMX tries to support all the POSIX-like -capability exposed by the EMX runtime, and mostly succeeds; :func:`fork` and +capability exposed by the EMX runtime, and mostly succeeds; :func:`!fork` and :func:`fcntl` are restricted by the limitations of the underlying emulation layer. The standard OS/2 port, which uses IBM's Visual Age compiler, also gained support for case-sensitive import semantics as part of the integration of @@ -1998,13 +1998,13 @@ Some of the more notable changes are: It would be difficult to detect any resulting difference from Python code, apart from a slight speed up when Python is run without :option:`-O`. - C extensions that access the :attr:`f_lineno` field of frame objects should + C extensions that access the :attr:`~frame.f_lineno` field of frame objects should instead call ``PyCode_Addr2Line(f->f_code, f->f_lasti)``. This will have the added effect of making the code work as desired under "python -O" in earlier versions of Python. A nifty new feature is that trace functions can now assign to the - :attr:`f_lineno` attribute of frame objects, changing the line that will be + :attr:`~frame.f_lineno` attribute of frame objects, changing the line that will be executed next. A ``jump`` command has been added to the :mod:`pdb` debugger taking advantage of this new feature. (Implemented by Richie Hindle.) @@ -2031,9 +2031,9 @@ code: the file's encoding (UTF-8, Latin-1, or whatever) by adding a comment to the top of the file. See section :ref:`section-encodings` for more information. -* Calling Tcl methods through :mod:`_tkinter` no longer returns only strings. +* Calling Tcl methods through :mod:`!_tkinter` no longer returns only strings. Instead, if Tcl returns other objects those objects are converted to their - Python equivalent, if one exists, or wrapped with a :class:`_tkinter.Tcl_Obj` + Python equivalent, if one exists, or wrapped with a :class:`!_tkinter.Tcl_Obj` object if no Python equivalent exists. * Large octal and hex literals such as ``0xffffffff`` now trigger a @@ -2049,10 +2049,10 @@ code: * You can no longer disable assertions by assigning to ``__debug__``. -* The Distutils :func:`setup` function has gained various new keyword arguments +* The Distutils :func:`!setup` function has gained various new keyword arguments such as *depends*. Old versions of the Distutils will abort if passed unknown keywords. A solution is to check for the presence of the new - :func:`get_distutil_options` function in your :file:`setup.py` and only uses the + :func:`!get_distutil_options` function in your :file:`setup.py` and only uses the new keywords with a version of the Distutils that supports them:: from distutils import core diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index cab321c3e54d18..e9a59f4a62551a 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -324,7 +324,8 @@ function, as previously described. In other words, ``@A @B @C(args)`` becomes:: Getting this right can be slightly brain-bending, but it's not too difficult. -A small related change makes the :attr:`func_name` attribute of functions +A small related change makes the :attr:`func_name ` +attribute of functions writable. This attribute is used to display function names in tracebacks, so decorators should change the name of any new function that's constructed and returned. @@ -994,7 +995,7 @@ fixes. Here's a partial list of the most notable changes, sorted alphabetically by module name. Consult the :file:`Misc/NEWS` file in the source tree for a more complete list of changes, or look through the CVS logs for all the details. -* The :mod:`asyncore` module's :func:`loop` function now has a *count* parameter +* The :mod:`!asyncore` module's :func:`!loop` function now has a *count* parameter that lets you perform a limited number of passes through the polling loop. The default is still to loop forever. @@ -1051,9 +1052,9 @@ complete list of changes, or look through the CVS logs for all the details. advantage of :class:`collections.deque` for improved performance. (Contributed by Raymond Hettinger.) -* The :mod:`ConfigParser` classes have been enhanced slightly. The :meth:`read` +* The :mod:`ConfigParser ` classes have been enhanced slightly. The :meth:`~configparser.ConfigParser.read` method now returns a list of the files that were successfully parsed, and the - :meth:`set` method raises :exc:`TypeError` if passed a *value* argument that + :meth:`~configparser.ConfigParser.set` method raises :exc:`TypeError` if passed a *value* argument that isn't a string. (Contributed by John Belmonte and David Goodger.) * The :mod:`curses` module now supports the ncurses extension @@ -1163,7 +1164,7 @@ complete list of changes, or look through the CVS logs for all the details. * A number of functions were added to the :mod:`locale` module, such as :func:`bind_textdomain_codeset` to specify a particular encoding and a family of - :func:`l\*gettext` functions that return messages in the chosen encoding. + :func:`!l\*gettext` functions that return messages in the chosen encoding. (Contributed by Gustavo Niemeyer.) * Some keyword arguments were added to the :mod:`logging` package's diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 64b951da3fd5b8..627c918dd6d8b4 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1167,10 +1167,10 @@ marked in the following list. * It's now illegal to mix iterating over a file with ``for line in file`` and calling the file object's :meth:`read`/:meth:`readline`/:meth:`readlines` - methods. Iteration uses an internal buffer and the :meth:`read\*` methods + methods. Iteration uses an internal buffer and the :meth:`!read\*` methods don't use that buffer. Instead they would return the data following the buffer, causing the data to appear out of order. Mixing iteration and these - methods will now trigger a :exc:`ValueError` from the :meth:`read\*` method. + methods will now trigger a :exc:`ValueError` from the :meth:`!read\*` method. (Implemented by Thomas Wouters.) .. Patch 1397960 diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 016de153f3dd6a..d947f61b50cfe0 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1677,8 +1677,9 @@ Some smaller changes made to the core Python language are: (:issue:`1591665`) * Instance method objects have new attributes for the object and function - comprising the method; the new synonym for :attr:`im_self` is - :attr:`__self__`, and :attr:`im_func` is also available as :attr:`__func__`. + comprising the method; the new synonym for :attr:`!im_self` is + :attr:`~method.__self__`, and :attr:`!im_func` is also available as + :attr:`~method.__func__`. The old names are still supported in Python 2.6, but are gone in 3.0. * An obscure change: when you use the :func:`locals` function inside a @@ -1788,7 +1789,7 @@ changes, sorted alphabetically by module name. Consult the :file:`Misc/NEWS` file in the source tree for a more complete list of changes, or look through the Subversion logs for all the details. -* The :mod:`asyncore` and :mod:`asynchat` modules are +* The :mod:`!asyncore` and :mod:`!asynchat` modules are being actively maintained again, and a number of patches and bugfixes were applied. (Maintained by Josiah Carlson; see :issue:`1736190` for one patch.) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index da66dd731831bc..81fe132d50e1f1 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -287,7 +287,7 @@ remains O(1). The standard library now supports use of ordered dictionaries in several modules. -* The :mod:`ConfigParser` module uses them by default, meaning that +* The :mod:`ConfigParser ` module uses them by default, meaning that configuration files can now be read, modified, and then written back in their original order. @@ -858,9 +858,11 @@ Some smaller changes made to the core Python language are: .. XXX bytearray doesn't seem to be documented -* When using ``@classmethod`` and ``@staticmethod`` to wrap +* When using :class:`@classmethod ` and + :class:`@staticmethod ` to wrap methods as class or static methods, the wrapper object now - exposes the wrapped function as their :attr:`__func__` attribute. + exposes the wrapped function as their :attr:`~method.__func__` + attribute. (Contributed by Amaury Forgeot d'Arc, after a suggestion by George Sakkis; :issue:`5982`.) @@ -1132,7 +1134,7 @@ changes, or look through the Subversion logs for all the details. another type that isn't a :class:`Mapping`. (Fixed by Daniel Stutzbach; :issue:`8729`.) -* Constructors for the parsing classes in the :mod:`ConfigParser` module now +* Constructors for the parsing classes in the :mod:`ConfigParser ` module now take an *allow_no_value* parameter, defaulting to false; if true, options without values will be allowed. For example:: @@ -1415,7 +1417,7 @@ changes, or look through the Subversion logs for all the details. :func:`~math.lgamma` for the natural log of the Gamma function. (Contributed by Mark Dickinson and nirinA raseliarison; :issue:`3366`.) -* The :mod:`multiprocessing` module's :class:`Manager*` classes +* The :mod:`multiprocessing` module's :class:`!Manager*` classes can now be passed a callable that will be called whenever a subprocess is started, along with a set of arguments that will be passed to the callable. @@ -2404,7 +2406,7 @@ Other Changes and Fixes :issue:`5464`.) * When importing a module from a :file:`.pyc` or :file:`.pyo` file - with an existing :file:`.py` counterpart, the :attr:`co_filename` + with an existing :file:`.py` counterpart, the :attr:`~codeobject.co_filename` attributes of the resulting code objects are overwritten when the original filename is obsolete. This can happen if the file has been renamed, moved, or is accessed through different paths. (Patch by diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index b0c2529e780213..89e12062abaddd 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -711,7 +711,7 @@ new powerful features added: {Exception}({args})` instead of :samp:`raise {Exception}, {args}`. Additionally, you can no longer explicitly specify a traceback; instead, if you *have* to do this, you can assign directly to the - :attr:`__traceback__` attribute (see below). + :attr:`~BaseException.__traceback__` attribute (see below). * :pep:`3110`: Catching exceptions. You must now use :samp:`except {SomeException} as {variable}` instead @@ -725,7 +725,7 @@ new powerful features added: handler block. This usually happens due to a bug in the handler block; we call this a *secondary* exception. In this case, the original exception (that was being handled) is saved as the - :attr:`__context__` attribute of the secondary exception. + :attr:`~BaseException.__context__` attribute of the secondary exception. Explicit chaining is invoked with this syntax:: raise SecondaryException() from primary_exception @@ -733,14 +733,15 @@ new powerful features added: (where *primary_exception* is any expression that produces an exception object, probably an exception that was previously caught). In this case, the primary exception is stored on the - :attr:`__cause__` attribute of the secondary exception. The + :attr:`~BaseException.__cause__` attribute of the secondary exception. The traceback printed when an unhandled exception occurs walks the chain - of :attr:`__cause__` and :attr:`__context__` attributes and prints a + of :attr:`!__cause__` and :attr:`~BaseException.__context__` attributes and + prints a separate traceback for each component of the chain, with the primary exception at the top. (Java users may recognize this behavior.) * :pep:`3134`: Exception objects now store their traceback as the - :attr:`__traceback__` attribute. This means that an exception + :attr:`~BaseException.__traceback__` attribute. This means that an exception object now contains all the information pertaining to an exception, and there are fewer reasons to use :func:`sys.exc_info` (though the latter is not removed). @@ -779,14 +780,15 @@ Operators And Special Methods * Removed support for :attr:`__members__` and :attr:`__methods__`. -* The function attributes named :attr:`func_X` have been renamed to - use the :data:`__X__` form, freeing up these names in the function +* The function attributes named :attr:`!func_X` have been renamed to + use the :attr:`!__X__` form, freeing up these names in the function attribute namespace for user-defined attributes. To wit, - :attr:`func_closure`, :attr:`func_code`, :attr:`func_defaults`, - :attr:`func_dict`, :attr:`func_doc`, :attr:`func_globals`, - :attr:`func_name` were renamed to :attr:`__closure__`, - :attr:`__code__`, :attr:`__defaults__`, :attr:`~object.__dict__`, - :attr:`__doc__`, :attr:`__globals__`, :attr:`~definition.__name__`, + :attr:`!func_closure`, :attr:`!func_code`, :attr:`!func_defaults`, + :attr:`!func_dict`, :attr:`!func_doc`, :attr:`!func_globals`, + :attr:`!func_name` were renamed to :attr:`~function.__closure__`, + :attr:`~function.__code__`, :attr:`~function.__defaults__`, + :attr:`~function.__dict__`, :attr:`~function.__doc__`, + :attr:`~function.__globals__`, :attr:`~function.__name__`, respectively. * :meth:`!__nonzero__` is now :meth:`~object.__bool__`. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index df821d68eb8d9f..cd86c82caffc56 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -399,10 +399,14 @@ PEP 626: Precise line numbers for debugging and other tools PEP 626 brings more precise and reliable line numbers for debugging, profiling and coverage tools. Tracing events, with the correct line number, are generated for all lines of code executed and only for lines of code that are executed. -The ``f_lineno`` attribute of frame objects will always contain the expected line number. +The :attr:`~frame.f_lineno` attribute of frame objects will always contain the +expected line number. -The ``co_lnotab`` attribute of code objects is deprecated and will be removed in 3.12. -Code that needs to convert from offset to line number should use the new ``co_lines()`` method instead. +The :attr:`~codeobject.co_lnotab` attribute of +:ref:`code objects ` is deprecated and +will be removed in 3.12. +Code that needs to convert from offset to line number should use the new +:meth:`~codeobject.co_lines` method instead. PEP 634: Structural Pattern Matching ------------------------------------ @@ -1276,7 +1280,7 @@ Add negative indexing support to :attr:`PurePath.parents (Contributed by Yaroslav Pankovych in :issue:`21041`.) Add :meth:`Path.hardlink_to ` method that -supersedes :meth:`~pathlib.Path.link_to`. The new method has the same argument +supersedes :meth:`!link_to`. The new method has the same argument order as :meth:`~pathlib.Path.symlink_to`. (Contributed by Barney Gale in :issue:`39950`.) @@ -1738,7 +1742,7 @@ Deprecated (Contributed by Jelle Zijlstra in :gh:`87889`.) -* :meth:`pathlib.Path.link_to` is deprecated and slated for removal in +* :meth:`!pathlib.Path.link_to` is deprecated and slated for removal in Python 3.12. Use :meth:`pathlib.Path.hardlink_to` instead. (Contributed by Barney Gale in :issue:`39950`.) @@ -1769,7 +1773,7 @@ Deprecated * NPN features like :meth:`ssl.SSLSocket.selected_npn_protocol` and :meth:`ssl.SSLContext.set_npn_protocols` are replaced by ALPN. -* The threading debug (:envvar:`PYTHONTHREADDEBUG` environment variable) is +* The threading debug (:envvar:`!PYTHONTHREADDEBUG` environment variable) is deprecated in Python 3.10 and will be removed in Python 3.12. This feature requires a :ref:`debug build of Python `. (Contributed by Victor Stinner in :issue:`44584`.) @@ -1959,11 +1963,11 @@ Changes in the C API source_buf = PyBytes_AsString(source_bytes_object); code = Py_CompileString(source_buf, filename, Py_file_input); - * For ``FrameObject`` objects, the ``f_lasti`` member now represents a wordcode + * For ``FrameObject`` objects, the :attr:`~frame.f_lasti` member now represents a wordcode offset instead of a simple offset into the bytecode string. This means that this number needs to be multiplied by 2 to be used with APIs that expect a byte offset instead (like :c:func:`PyCode_Addr2Line` for example). Notice as well that the - ``f_lasti`` member of ``FrameObject`` objects is not considered stable: please + :attr:`!f_lasti` member of ``FrameObject`` objects is not considered stable: please use :c:func:`PyFrame_GetLineNumber` instead. CPython bytecode changes diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 48a0e621baad02..cb646a54df3607 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1747,7 +1747,7 @@ Modules (Contributed by Brett Cannon in :issue:`47061` and Victor Stinner in :gh:`68966`.) -* The :mod:`asynchat`, :mod:`asyncore` and :mod:`smtpd` modules have been +* The :mod:`!asynchat`, :mod:`!asyncore` and :mod:`!smtpd` modules have been deprecated since at least Python 3.6. Their documentation and deprecation warnings have now been updated to note they will be removed in Python 3.12. (Contributed by Hugo van Kemenade in :issue:`47022`.) @@ -1773,7 +1773,7 @@ Standard Library * the :class:`!configparser.SafeConfigParser` class * the :attr:`!configparser.ParsingError.filename` property - * the :meth:`configparser.RawConfigParser.readfp` method + * the :meth:`!configparser.RawConfigParser.readfp` method (Contributed by Hugo van Kemenade in :issue:`45173`.) @@ -1860,7 +1860,7 @@ Standard Library (Contributed by Erlend E. Aasland in :issue:`5846`.) -* :meth:`~!unittest.TestProgram.usageExit` is marked deprecated, to be removed +* :meth:`!unittest.TestProgram.usageExit` is marked deprecated, to be removed in 3.13. (Contributed by Carlos Damázio in :gh:`67048`.) @@ -1877,8 +1877,8 @@ and will be removed in Python 3.12. C APIs pending removal are :ref:`listed separately `. -* The :mod:`asynchat` module -* The :mod:`asyncore` module +* The :mod:`!asynchat` module +* The :mod:`!asyncore` module * The :ref:`entire distutils package ` * The :mod:`!imp` module * The :class:`typing.io ` namespace @@ -1902,10 +1902,10 @@ C APIs pending removal are * :func:`!importlib.util.set_package_wrapper` * :class:`!pkgutil.ImpImporter` * :class:`!pkgutil.ImpLoader` -* :meth:`pathlib.Path.link_to` +* :meth:`!pathlib.Path.link_to` * :func:`!sqlite3.enable_shared_cache` * :func:`!sqlite3.OptimizedUnicode` -* :envvar:`PYTHONTHREADDEBUG` environment variable +* :envvar:`!PYTHONTHREADDEBUG` environment variable * The following deprecated aliases in :mod:`unittest`: ============================ =============================== =============== @@ -2007,7 +2007,7 @@ Removed C APIs are :ref:`listed separately `. because it was not used and added by mistake in previous versions. (Contributed by Nikita Sobolev in :issue:`46483`.) -* Removed the :class:`!MailmanProxy` class in the :mod:`smtpd` module, +* Removed the :class:`!MailmanProxy` class in the :mod:`!smtpd` module, as it is unusable without the external :mod:`!mailman` package. (Contributed by Donghee Na in :issue:`35800`.) @@ -2458,11 +2458,12 @@ Porting to Python 3.11 * ``f_valuestack``: removed. The Python frame object is now created lazily. A side effect is that the - ``f_back`` member must not be accessed directly, since its value is now also + :attr:`~frame.f_back` member must not be accessed directly, + since its value is now also computed lazily. The :c:func:`PyFrame_GetBack` function must be called instead. - Debuggers that accessed the ``f_locals`` directly *must* call + Debuggers that accessed the :attr:`~frame.f_locals` directly *must* call :c:func:`PyFrame_GetLocals` instead. They no longer need to call :c:func:`PyFrame_FastToLocalsWithError` or :c:func:`PyFrame_LocalsToFast`, in fact they should not call those functions. The necessary updating of the diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a4b3a6d12970b4..9a2ccf7ebc6a68 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -343,7 +343,9 @@ cores. This is currently only available through the C-API, though a Python API is :pep:`anticipated for 3.13 <554>`. Use the new :c:func:`Py_NewInterpreterFromConfig` function to -create an interpreter with its own GIL:: +create an interpreter with its own GIL: + +.. code-block:: c PyInterpreterConfig config = { .check_multi_interp_extensions = 1, @@ -838,8 +840,7 @@ shutil * :func:`shutil.rmtree` now accepts a new argument *onexc* which is an error handler like *onerror* but which expects an exception instance - rather than a *(typ, val, tb)* triplet. *onerror* is deprecated and - will be removed in Python 3.14. + rather than a *(typ, val, tb)* triplet. *onerror* is deprecated. (Contributed by Irit Katriel in :gh:`102828`.) * :func:`shutil.which` now consults the *PATHEXT* environment variable to @@ -1261,8 +1262,8 @@ Deprecated :mod:`concurrent.futures` the fix is to use a different :mod:`multiprocessing` start method such as ``"spawn"`` or ``"forkserver"``. -* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated and will be removed - in Python 3.14. Use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) +* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated; + use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) * :mod:`sqlite3`: @@ -1324,7 +1325,8 @@ Deprecated ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) -* Accessing ``co_lnotab`` on code objects was deprecated in Python 3.10 via :pep:`626`, +* Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in + Python 3.10 via :pep:`626`, but it only got a proper :exc:`DeprecationWarning` in 3.12, therefore it will be removed in 3.14. (Contributed by Nikita Sobolev in :gh:`101866`.) @@ -1431,7 +1433,7 @@ and will be removed in Python 3.14. * The ``__package__`` and ``__cached__`` attributes on module objects. -* The ``co_lnotab`` attribute of code objects. +* The :attr:`~codeobject.co_lnotab` attribute of code objects. Pending Removal in Python 3.15 ------------------------------ @@ -1640,7 +1642,10 @@ locale use :func:`locale.format_string` instead. (Contributed by Victor Stinner in :gh:`94226`.) -* ``smtpd``: The module has been removed according to the schedule in :pep:`594`, +smtpd +----- + +* The ``smtpd`` module has been removed according to the schedule in :pep:`594`, having been deprecated in Python 3.4.7 and 3.5.4. Use aiosmtpd_ PyPI module or any other :mod:`asyncio`-based server instead. @@ -1892,6 +1897,15 @@ Changes in the Python API * Mixing tabs and spaces as indentation in the same file is not supported anymore and will raise a :exc:`TabError`. +* The :mod:`threading` module now expects the :mod:`!_thread` module to have + an ``_is_main_interpreter`` attribute. It is a function with no + arguments that returns ``True`` if the current interpreter is the + main interpreter. + + Any library or application that provides a custom ``_thread`` module + should provide ``_is_main_interpreter()``. + (See :gh:`112826`.) + Build Changes ============= diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 3fd0f5e165f018..3ab6d1ddc6ef21 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -85,7 +85,14 @@ Important deprecations, removals or restrictions: New Features ============ +Improved Error Messages +----------------------- +* The interpreter now colorizes error messages when displaying tracebacks by default. + This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment + variable as well as the canonical ``NO_COLOR`` and ``FORCE_COLOR`` environment + variables. See also :ref:`using-on-controlling-color`. + (Contributed by Pablo Galindo Salgado in :gh:`112730`.) Other Language Changes ====================== @@ -192,6 +199,27 @@ doctest :attr:`doctest.TestResults.skipped` attributes. (Contributed by Victor Stinner in :gh:`108794`.) +email +----- + +* :func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now return + ``('', '')`` 2-tuples in more situations where invalid email addresses are + encountered instead of potentially inaccurate values. Add optional *strict* + parameter to these two functions: use ``strict=False`` to get the old + behavior, accept malformed inputs. + ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to + check if the *strict* paramater is available. + (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve + the CVE-2023-27043 fix.) + +fractions +--------- + +* Formatting for objects of type :class:`fractions.Fraction` now supports + the standard format specification mini-language rules for fill, alignment, + sign handling, minimum width and grouping. (Contributed by Mark Dickinson + in :gh:`111320`) + glob ---- @@ -255,6 +283,25 @@ os CPU resources of a container system without having to modify the container (application code). (Contributed by Donghee Na in :gh:`109595`) +* Add support of :func:`os.lchmod` and the *follow_symlinks* argument + in :func:`os.chmod` on Windows. + Note that the default value of *follow_symlinks* in :func:`!os.lchmod` is + ``False`` on Windows. + (Contributed by Serhiy Storchaka in :gh:`59616`) + +* Add support of :func:`os.fchmod` and a file descriptor + in :func:`os.chmod` on Windows. + (Contributed by Serhiy Storchaka in :gh:`113191`) + +* :func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned + process use the current process environment. + (Contributed by Jakub Kulik in :gh:`113119`.) + +* :func:`os.posix_spawn` gains an :attr:`os.POSIX_SPAWN_CLOSEFROM` attribute for + use in ``file_actions=`` on platforms that support + :c:func:`!posix_spawn_file_actions_addclosefrom_np`. + (Contributed by Jakub Kulik in :gh:`113117`.) + pathlib ------- @@ -270,9 +317,11 @@ pathlib (Contributed by Barney Gale in :gh:`73435`.) * Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.glob`, - :meth:`~pathlib.Path.rglob`, :meth:`~pathlib.Path.is_file`, and - :meth:`~pathlib.Path.is_dir`. - (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`.) + :meth:`~pathlib.Path.rglob`, :meth:`~pathlib.Path.is_file`, + :meth:`~pathlib.Path.is_dir`, :meth:`~pathlib.Path.owner`, + :meth:`~pathlib.Path.group`. + (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`, and + Kamil Turek in :gh:`107962`). pdb --- @@ -285,6 +334,16 @@ pdb identified and executed. (Contributed by Tian Gao in :gh:`108464`.) +* ``sys.path[0]`` will no longer be replaced by the directory of the script + being debugged when ``sys.flags.safe_path`` is set (via the :option:`-P` + command line option or :envvar:`PYTHONSAFEPATH` environment variable). + (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) + +re +-- +* Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. + :exc:`!re.error` is kept for backward compatibility. + sqlite3 ------- @@ -292,6 +351,28 @@ sqlite3 object is not :meth:`closed ` explicitly. (Contributed by Erlend E. Aasland in :gh:`105539`.) +subprocess +---------- + +* The :mod:`subprocess` module now uses the :func:`os.posix_spawn` function in + more situations. Notably in the default case of ``close_fds=True`` on more + recent versions of platforms including Linux, FreeBSD, and Solaris where the + C library provides :c:func:`!posix_spawn_file_actions_addclosefrom_np`. + On Linux this should perform similar to our existing Linux :c:func:`!vfork` + based code. A private control knob :attr:`!subprocess._USE_POSIX_SPAWN` can + be set to ``False`` if you need to force :mod:`subprocess` not to ever use + :func:`os.posix_spawn`. Please report your reason and platform details in + the CPython issue tracker if you set this so that we can improve our API + selection logic for everyone. + (Contributed by Jakub Kulik in :gh:`113117`.) + +sys +--- + +* Add the :func:`sys._is_interned` function to test if the string was interned. + This function is not guaranteed to exist in all implementations of Python. + (Contributed by Serhiy Storchaka in :gh:`78573`.) + tkinter ------- @@ -301,6 +382,11 @@ tkinter :meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. (Contributed by Miguel, klappnase and Serhiy Storchaka in :gh:`72684`.) +* Add support of the "vsapi" element type in + the :meth:`~tkinter.ttk.Style.element_create` method of + :class:`tkinter.ttk.Style`. + (Contributed by Serhiy Storchaka in :gh:`68166`.) + traceback --------- @@ -308,6 +394,12 @@ traceback to format the nested exceptions of a :exc:`BaseExceptionGroup` instance, recursively. (Contributed by Irit Katriel in :gh:`105292`.) +* Add the field *exc_type_str* to :class:`~traceback.TracebackException`, which + holds a string display of the *exc_type*. Deprecate the field *exc_type* + which holds the type object itself. Add parameter *save_exc_type* (default + ``True``) to indicate whether ``exc_type`` should be saved. + (Contributed by Irit Katriel in :gh:`112332`.) + typing ------ @@ -332,12 +424,27 @@ venv (using ``--without-scm-ignore-files``). (Contributed by Brett Cannon in :gh:`108125`.) +warnings +-------- + +* The new :func:`warnings.deprecated` decorator provides a way to communicate + deprecations to :term:`static type checkers ` and + to warn on usage of deprecated classes and functions. A runtime deprecation + warning may also be emitted when a decorated function or class is used at runtime. + See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) + Optimizations ============= * :func:`textwrap.indent` is now ~30% faster than before for large input. (Contributed by Inada Naoki in :gh:`107369`.) +* The :mod:`subprocess` module uses :func:`os.posix_spawn` in more situations + including the default where ``close_fds=True`` on many modern platforms. This + should provide a noteworthy performance increase launching processes on + FreeBSD and Solaris. See the ``subprocess`` section above for details. + (Contributed by Jakub Kulik in :gh:`113117`.) + Deprecated ========== @@ -367,6 +474,15 @@ Deprecated security and functionality bugs. This includes removal of the ``--cgi`` flag to the ``python -m http.server`` command line in 3.15. +* :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function. + Replace it with :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. + (Contributed by Inada Naoki in :gh:`73427`.) + +* :mod:`traceback`: + + * The field *exc_type* of :class:`traceback.TracebackException` is + deprecated. Use *exc_type_str* instead. + * :mod:`typing`: * Creating a :class:`typing.NamedTuple` class using keyword arguments to denote @@ -438,12 +554,15 @@ Deprecated (as has always been the case for an executing frame). (Contributed by Irit Katriel in :gh:`79932`.) -* Assignment to a function's ``__code__`` attribute where the new code +* Assignment to a function's :attr:`~function.__code__` attribute where the new code object's type does not match the function's type, is deprecated. The different types are: plain function, generator, async generator and coroutine. (Contributed by Irit Katriel in :gh:`81137`.) +* Deprecate undocumented :func:`!pydoc.ispackage` function. + (Contributed by Zackery Spytz in :gh:`64020`.) + Pending Removal in Python 3.14 ------------------------------ @@ -533,7 +652,8 @@ Pending Removal in Python 3.14 * date and datetime adapter, date and timestamp converter: see the :mod:`sqlite3` documentation for suggested replacement recipes. -* :class:`types.CodeType`: Accessing ``co_lnotab`` was deprecated in :pep:`626` +* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + deprecated in :pep:`626` since 3.10 and was planned to be removed in 3.12, but it only got a proper :exc:`DeprecationWarning` in 3.12. May be removed in 3.14. @@ -696,7 +816,8 @@ although there is currently no date scheduled for their removal. * :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. -* ``types.CodeType.co_lnotab``: use the ``co_lines`` attribute instead. +* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method + instead. * :class:`typing.Text` (:gh:`92332`). @@ -892,6 +1013,10 @@ importlib for migration advice. (Contributed by Jason R. Coombs in :gh:`106532`.) +* Remove deprecated :meth:`~object.__getitem__` access for + :class:`!importlib.metadata.EntryPoint` objects. + (Contributed by Jason R. Coombs in :gh:`113175`.) + locale ------ @@ -1032,6 +1157,23 @@ Changes in the Python API recomended in the documentation. (Contributed by Serhiy Storchaka in :gh:`106672`.) +* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to + retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or + :exc:`KeyError` on Unix platforms where the password database is empty. + +* The :mod:`threading` module now expects the :mod:`!_thread` module to have + an ``_is_main_interpreter`` attribute. It is a function with no + arguments that returns ``True`` if the current interpreter is the + main interpreter. + + Any library or application that provides a custom ``_thread`` module + must provide ``_is_main_interpreter()``, just like the module's + other "private" attributes. + (See :gh:`112826`.) + +* :class:`mailbox.Maildir` now ignores files with a leading dot. + (Contributed by Zackery Spytz in :gh:`65559`.) + Build Changes ============= @@ -1070,6 +1212,16 @@ New Features APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. (Contributed by Inada Naoki in :gh:`104922`.) +* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and + :c:func:`PyArg_VaParseTupleAndKeywords` has now type :c:expr:`char * const *` + in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. + It makes these functions compatible with arguments of type + :c:expr:`const char * const *`, :c:expr:`const char **` or + :c:expr:`char * const *` in C++ and :c:expr:`char * const *` in C + without an explicit type cast. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + (Contributed by Serhiy Storchaka in :gh:`65210`.) + * Add :c:func:`PyImport_AddModuleRef`: similar to :c:func:`PyImport_AddModule`, but return a :term:`strong reference` instead of a :term:`borrowed reference`. @@ -1196,6 +1348,9 @@ New Features :exc:`KeyError` if the key missing. (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) +* Add :c:func:`Py_HashPointer` function to hash a pointer. + (Contributed by Victor Stinner in :gh:`111545`.) + Porting to Python 3.13 ---------------------- diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index df32b76b6d7b03..9834bc03dc4b74 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -183,7 +183,7 @@ PEP 391: Dictionary Based Configuration for Logging The :mod:`logging` module provided two kinds of configuration, one style with function calls for each option or another style driven by an external file saved -in a :mod:`ConfigParser` format. Those options did not provide the flexibility +in a :mod:`configparser` format. Those options did not provide the flexibility to create configurations from JSON or YAML files, nor did they support incremental configuration, which is needed for specifying logger options from a command line. @@ -791,8 +791,9 @@ functools * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute pointing to the original callable function. This allows wrapped functions to - be introspected. It also copies :attr:`__annotations__` if defined. And now - it also gracefully skips over missing attributes such as :attr:`__doc__` which + be introspected. It also copies :attr:`~function.__annotations__` if + defined. And now it also gracefully skips over missing attributes such as + :attr:`~function.__doc__` which might not be defined for the wrapped callable. In the above example, the cache can be removed by recovering the original @@ -1857,12 +1858,12 @@ structure. asyncore -------- -:class:`asyncore.dispatcher` now provides a -:meth:`~asyncore.dispatcher.handle_accepted()` method +:class:`!asyncore.dispatcher` now provides a +:meth:`!handle_accepted()` method returning a ``(sock, addr)`` pair which is called when a connection has actually been established with a new remote endpoint. This is supposed to be used as a -replacement for old :meth:`~asyncore.dispatcher.handle_accept()` and avoids -the user to call :meth:`~asyncore.dispatcher.accept()` directly. +replacement for old :meth:`!handle_accept()` and avoids +the user to call :meth:`!accept()` directly. (Contributed by Giampaolo Rodolà; :issue:`6706`.) @@ -2133,7 +2134,7 @@ configparser The :mod:`configparser` module was modified to improve usability and predictability of the default parser and its supported INI syntax. The old -:class:`ConfigParser` class was removed in favor of :class:`SafeConfigParser` +:class:`!ConfigParser` class was removed in favor of :class:`!SafeConfigParser` which has in turn been renamed to :class:`~configparser.ConfigParser`. Support for inline comments is now turned off by default and section or option duplicates are not allowed in a single configuration source. @@ -2413,7 +2414,7 @@ when one operand is much larger than the other (patch by Andress Bennetts in (:issue:`1569291` by Alexander Belopolsky). The :class:`BaseHTTPRequestHandler` has more efficient buffering (:issue:`3709` by Andrew Schaaf). The :func:`operator.attrgetter` function has been sped-up (:issue:`10160` by -Christos Georgiou). And :class:`ConfigParser` loads multi-line arguments a bit +Christos Georgiou). And :class:`~configparser.ConfigParser` loads multi-line arguments a bit faster (:issue:`7113` by Łukasz Langa). @@ -2613,8 +2614,8 @@ This section lists previously described changes and other bugfixes that may require changes to your code: * The :mod:`configparser` module has a number of clean-ups. The major change is - to replace the old :class:`ConfigParser` class with long-standing preferred - alternative :class:`SafeConfigParser`. In addition there are a number of + to replace the old :class:`!ConfigParser` class with long-standing preferred + alternative :class:`!SafeConfigParser`. In addition there are a number of smaller incompatibilities: * The interpolation syntax is now validated on @@ -2736,8 +2737,8 @@ require changes to your code: thread-state aware APIs (such as :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread`) should be used instead. -* Due to security risks, :func:`asyncore.handle_accept` has been deprecated, and - a new function, :func:`asyncore.handle_accepted`, was added to replace it. +* Due to security risks, :func:`!asyncore.handle_accept` has been deprecated, and + a new function, :func:`!asyncore.handle_accepted`, was added to replace it. (Contributed by Giampaolo Rodola in :issue:`6706`.) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 5674bc7f359b72..760324ae66a3af 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1052,7 +1052,7 @@ their ``__init__`` method (for example, file objects) or in their crypt ----- -Addition of salt and modular crypt format (hashing method) and the :func:`~!crypt.mksalt` +Addition of salt and modular crypt format (hashing method) and the :func:`!mksalt` function to the :mod:`!crypt` module. (:issue:`10924`) @@ -1845,7 +1845,7 @@ signal smtpd ----- -The :mod:`smtpd` module now supports :rfc:`5321` (extended SMTP) and :rfc:`1870` +The :mod:`!smtpd` module now supports :rfc:`5321` (extended SMTP) and :rfc:`1870` (size extension). Per the standard, these extensions are enabled if and only if the client initiates the session with an ``EHLO`` command. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 2ddab76814369e..e07eda985d9bad 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -605,15 +605,15 @@ Using ``ABC`` as a base class has essentially the same effect as specifying aifc ---- -The :meth:`~!aifc.aifc.getparams` method now returns a namedtuple rather than a +The :meth:`!getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`17818`.) :func:`!aifc.open` now supports the context management protocol: when used in a -:keyword:`with` block, the :meth:`~!aifc.aifc.close` method of the returned +:keyword:`with` block, the :meth:`!close` method of the returned object will be called automatically at the end of the block. (Contributed by Serhiy Storchacha in :issue:`16486`.) -The :meth:`~!aifc.aifc.writeframesraw` and :meth:`~!aifc.aifc.writeframes` +The :meth:`!writeframesraw` and :meth:`!writeframes` methods now accept any :term:`bytes-like object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) @@ -632,7 +632,7 @@ audioop :mod:`!audioop` now supports 24-bit samples. (Contributed by Serhiy Storchaka in :issue:`12866`.) -New :func:`~!audioop.byteswap` function converts big-endian samples to +New :func:`!byteswap` function converts big-endian samples to little-endian and vice versa. (Contributed by Serhiy Storchaka in :issue:`19641`.) @@ -1369,9 +1369,9 @@ error. (Contributed by Atsuo Ishimoto and Hynek Schlawack in smtpd ----- -The :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now +The :class:`!SMTPServer` and :class:`!SMTPChannel` classes now accept a *map* keyword argument which, if specified, is passed in to -:class:`asynchat.async_chat` as its *map* argument. This allows an application +:class:`!asynchat.async_chat` as its *map* argument. This allows an application to avoid affecting the global socket map. (Contributed by Vinay Sajip in :issue:`11959`.) @@ -1528,7 +1528,7 @@ work on Windows. This change was actually inadvertently made in 3.3.4. sunau ----- -The :meth:`~!sunau.getparams` method now returns a namedtuple rather than a +The :meth:`!getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) :meth:`!sunau.open` now supports the context management protocol: when used in a @@ -1540,8 +1540,8 @@ in :issue:`18878`.) support for writing 24 sample using the module. (Contributed by Serhiy Storchaka in :issue:`19261`.) -The :meth:`~!sunau.AU_write.writeframesraw` and -:meth:`~!sunau.AU_write.writeframes` methods now accept any :term:`bytes-like +The :meth:`!writeframesraw` and +:meth:`!writeframes` methods now accept any :term:`bytes-like object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) @@ -1936,7 +1936,7 @@ Other Improvements * The :ref:`python ` command has a new :ref:`option `, ``-I``, which causes it to run in "isolated mode", which means that :data:`sys.path` contains neither the script's directory nor - the user's ``site-packages`` directory, and all :envvar:`PYTHON*` environment + the user's ``site-packages`` directory, and all :envvar:`!PYTHON*` environment variables are ignored (it implies both ``-s`` and ``-E``). Other restrictions may also be applied in the future, with the goal being to isolate the execution of a script from the user's environment. This is @@ -2370,7 +2370,7 @@ Changes in the Python API :issue:`18011`.) Note: this change was also inadvertently applied in Python 3.3.3. -* The :attr:`~cgi.FieldStorage.file` attribute is now automatically closed when +* The :attr:`!file` attribute is now automatically closed when the creating :class:`!cgi.FieldStorage` instance is garbage collected. If you were pulling the file object out separately from the :class:`!cgi.FieldStorage` instance and not keeping the instance alive, then you should either store the @@ -2405,8 +2405,8 @@ Changes in the Python API storage). (:issue:`17094`.) * Parameter names in ``__annotations__`` dicts are now mangled properly, - similarly to ``__kwdefaults__``. (Contributed by Yury Selivanov in - :issue:`20625`.) + similarly to :attr:`~function.__kwdefaults__`. + (Contributed by Yury Selivanov in :issue:`20625`.) * :attr:`hashlib.hash.name` now always returns the identifier in lower case. Previously some builtin hashes had uppercase names, but now that it is a diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index ae6affcab664c6..1c7a9270af0aab 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -878,7 +878,7 @@ size of decompressed data. (Contributed by Nikolaus Rath in :issue:`15955`.) cgi --- -The :class:`~cgi.FieldStorage` class now supports the :term:`context manager` +The :class:`!FieldStorage` class now supports the :term:`context manager` protocol. (Contributed by Berker Peksag in :issue:`20289`.) @@ -1252,7 +1252,7 @@ Oberkirch in :issue:`21800`.) imghdr ------ -The :func:`~!imghdr.what` function now recognizes the +The :func:`!what` function now recognizes the `OpenEXR `_ format (contributed by Martin Vignali and Claudiu Popa in :issue:`20295`), and the `WebP `_ format @@ -1663,34 +1663,34 @@ during debugging, instead of integer "magic numbers". smtpd ----- -Both the :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now +Both the :class:`!SMTPServer` and :class:`!SMTPChannel` classes now accept a *decode_data* keyword argument to determine if the ``DATA`` portion of the SMTP transaction is decoded using the ``"utf-8"`` codec or is instead provided to the -:meth:`SMTPServer.process_message() ` +:meth:`!SMTPServer.process_message()` method as a byte string. The default is ``True`` for backward compatibility reasons, but will change to ``False`` in Python 3.6. If *decode_data* is set to ``False``, the ``process_message`` method must be prepared to accept keyword arguments. (Contributed by Maciej Szulik in :issue:`19662`.) -The :class:`~smtpd.SMTPServer` class now advertises the ``8BITMIME`` extension +The :class:`!SMTPServer` class now advertises the ``8BITMIME`` extension (:rfc:`6152`) if *decode_data* has been set ``True``. If the client specifies ``BODY=8BITMIME`` on the ``MAIL`` command, it is passed to -:meth:`SMTPServer.process_message() ` +:meth:`!SMTPServer.process_message()` via the *mail_options* keyword. (Contributed by Milan Oberkirch and R. David Murray in :issue:`21795`.) -The :class:`~smtpd.SMTPServer` class now also supports the ``SMTPUTF8`` +The :class:`!SMTPServer` class now also supports the ``SMTPUTF8`` extension (:rfc:`6531`: Internationalized Email). If the client specified ``SMTPUTF8 BODY=8BITMIME`` on the ``MAIL`` command, they are passed to -:meth:`SMTPServer.process_message() ` +:meth:`!SMTPServer.process_message()` via the *mail_options* keyword. It is the responsibility of the ``process_message`` method to correctly handle the ``SMTPUTF8`` data. (Contributed by Milan Oberkirch in :issue:`21725`.) It is now possible to provide, directly or via name resolution, IPv6 -addresses in the :class:`~smtpd.SMTPServer` constructor, and have it +addresses in the :class:`!SMTPServer` constructor, and have it successfully connect. (Contributed by Milan Oberkirch in :issue:`14758`.) @@ -1714,7 +1714,7 @@ support :rfc:`6531` (SMTPUTF8). sndhdr ------ -The :func:`~sndhdr.what` and :func:`~sndhdr.whathdr` functions now return +The :func:`!what` and :func:`!whathdr` functions now return a :func:`~collections.namedtuple`. (Contributed by Claudiu Popa in :issue:`18615`.) @@ -1947,7 +1947,8 @@ traceback --------- New :func:`~traceback.walk_stack` and :func:`~traceback.walk_tb` -functions to conveniently traverse frame and traceback objects. +functions to conveniently traverse frame and +:ref:`traceback objects `. (Contributed by Robert Collins in :issue:`17911`.) New lightweight classes: :class:`~traceback.TracebackException`, @@ -2295,9 +2296,9 @@ slated for removal in Python 3.6. The :func:`asyncio.async` function is deprecated in favor of :func:`~asyncio.ensure_future`. -The :mod:`smtpd` module has in the past always decoded the DATA portion of +The :mod:`!smtpd` module has in the past always decoded the DATA portion of email messages using the ``utf-8`` codec. This can now be controlled by the -new *decode_data* keyword to :class:`~smtpd.SMTPServer`. The default value is +new *decode_data* keyword to :class:`!SMTPServer`. The default value is ``True``, but this default is deprecated. Specify the *decode_data* keyword with an appropriate value to avoid the deprecation warning. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index c15d8be651fd17..11e1d73232a96d 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -1961,14 +1961,14 @@ Deprecated Python modules, functions and methods asynchat ~~~~~~~~ -The :mod:`asynchat` has been deprecated in favor of :mod:`asyncio`. +The :mod:`!asynchat` has been deprecated in favor of :mod:`asyncio`. (Contributed by Mariatta in :issue:`25002`.) asyncore ~~~~~~~~ -The :mod:`asyncore` has been deprecated in favor of :mod:`asyncio`. +The :mod:`!asyncore` has been deprecated in favor of :mod:`asyncio`. (Contributed by Mariatta in :issue:`25002`.) @@ -2160,14 +2160,15 @@ Changes in the Python API * :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** argument is not set. Previously only ``NULL`` was returned. -* The format of the ``co_lnotab`` attribute of code objects changed to support +* The format of the :attr:`~codeobject.co_lnotab` attribute of code objects + changed to support a negative line number delta. By default, Python does not emit bytecode with - a negative line number delta. Functions using ``frame.f_lineno``, + a negative line number delta. Functions using :attr:`frame.f_lineno`, ``PyFrame_GetLineNumber()`` or ``PyCode_Addr2Line()`` are not affected. - Functions directly decoding ``co_lnotab`` should be updated to use a signed + Functions directly decoding :attr:`!co_lnotab` should be updated to use a signed 8-bit integer type for the line number delta, but this is only required to support applications using a negative line number delta. See - ``Objects/lnotab_notes.txt`` for the ``co_lnotab`` format and how to decode + ``Objects/lnotab_notes.txt`` for the :attr:`!co_lnotab` format and how to decode it, and see the :pep:`511` for the rationale. * The functions in the :mod:`compileall` module now return booleans instead @@ -2188,7 +2189,7 @@ Changes in the Python API :mod:`calendar`, :mod:`!cgi`, :mod:`csv`, :mod:`~xml.etree.ElementTree`, :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, :mod:`mailbox`, - :mod:`mimetypes`, :mod:`optparse`, :mod:`plistlib`, :mod:`smtpd`, + :mod:`mimetypes`, :mod:`optparse`, :mod:`plistlib`, :mod:`!smtpd`, :mod:`subprocess`, :mod:`tarfile`, :mod:`threading` and :mod:`wave`. This means they will export new symbols when ``import *`` is used. @@ -2218,11 +2219,11 @@ Changes in the Python API an error (e.g. ``EBADF``) was reported by the underlying system call. (Contributed by Martin Panter in :issue:`26685`.) -* The *decode_data* argument for the :class:`smtpd.SMTPChannel` and - :class:`smtpd.SMTPServer` constructors is now ``False`` by default. +* The *decode_data* argument for the :class:`!smtpd.SMTPChannel` and + :class:`!smtpd.SMTPServer` constructors is now ``False`` by default. This means that the argument passed to - :meth:`~smtpd.SMTPServer.process_message` is now a bytes object by - default, and ``process_message()`` will be passed keyword arguments. + :meth:`!process_message` is now a bytes object by + default, and :meth:`!process_message` will be passed keyword arguments. Code that has already been updated in accordance with the deprecation warning generated by 3.5 will not be affected. diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index a7d5c3db6ddcb2..402b15a277e53d 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -525,8 +525,8 @@ Other Language Changes * In order to better support dynamic creation of stack traces, :class:`types.TracebackType` can now be instantiated from Python code, and - the ``tb_next`` attribute on :ref:`tracebacks ` is now - writable. + the :attr:`~traceback.tb_next` attribute on + :ref:`tracebacks ` is now writable. (Contributed by Nathaniel J. Smith in :issue:`30579`.) * When using the :option:`-m` switch, ``sys.path[0]`` is now eagerly expanded @@ -851,7 +851,7 @@ crypt The :mod:`!crypt` module now supports the Blowfish hashing method. (Contributed by Serhiy Storchaka in :issue:`31664`.) -The :func:`~!crypt.mksalt` function now allows specifying the number of rounds +The :func:`!mksalt` function now allows specifying the number of rounds for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) @@ -1891,7 +1891,7 @@ Other CPython Implementation Changes * Trace hooks may now opt out of receiving the ``line`` and opt into receiving the ``opcode`` events from the interpreter by setting the corresponding new - ``f_trace_lines`` and ``f_trace_opcodes`` attributes on the + :attr:`~frame.f_trace_lines` and :attr:`~frame.f_trace_opcodes` attributes on the frame being traced. (Contributed by Nick Coghlan in :issue:`31344`.) * Fixed some consistency problems with namespace package module attributes. @@ -2004,15 +2004,15 @@ importlib --------- Methods -:meth:`MetaPathFinder.find_module() ` +:meth:`!MetaPathFinder.find_module()` (replaced by :meth:`MetaPathFinder.find_spec() `) and -:meth:`PathEntryFinder.find_loader() ` +:meth:`!PathEntryFinder.find_loader()` (replaced by :meth:`PathEntryFinder.find_spec() `) both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. -(Contributed by Matthias Bussonnier in :issue:`29576`) +(Contributed by Matthias Bussonnier in :issue:`29576`.) The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in favour of :class:`importlib.abc.ResourceReader`. @@ -2144,9 +2144,9 @@ The following features and APIs have been removed from Python 3.7: * Removed support of the *exclude* argument in :meth:`tarfile.TarFile.add`. It was deprecated in Python 2.7 and 3.2. Use the *filter* argument instead. -* The ``splitunc()`` function in the :mod:`ntpath` module was deprecated in - Python 3.1, and has now been removed. Use the :func:`~os.path.splitdrive` - function instead. +* The :func:`!ntpath.splitunc` function was deprecated in + Python 3.1, and has now been removed. Use :func:`~os.path.splitdrive` + instead. * :func:`collections.namedtuple` no longer supports the *verbose* parameter or ``_source`` attribute which showed the generated source code for the @@ -2304,9 +2304,9 @@ Changes in the Python API * The :attr:`struct.Struct.format` type is now :class:`str` instead of :class:`bytes`. (Contributed by Victor Stinner in :issue:`21071`.) -* :func:`~cgi.parse_multipart` now accepts the *encoding* and *errors* +* :func:`!cgi.parse_multipart` now accepts the *encoding* and *errors* arguments and returns the same results as - :class:`~FieldStorage`: for non-file fields, the value associated to a key + :class:`!FieldStorage`: for non-file fields, the value associated to a key is a list of strings, not bytes. (Contributed by Pierre Quentel in :issue:`29979`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 4574702b1a600f..d373fa163ff737 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -123,7 +123,7 @@ There is a new function parameter syntax ``/`` to indicate that some function parameters must be specified positionally and cannot be used as keyword arguments. This is the same notation shown by ``help()`` for C functions annotated with Larry Hastings' -`Argument Clinic `__ tool. +`Argument Clinic `__ tool. In the following example, parameters *a* and *b* are positional-only, while *c* or *d* can be positional or keyword, and *e* or *f* are @@ -1086,7 +1086,7 @@ pathlib contain characters unrepresentable at the OS level. (Contributed by Serhiy Storchaka in :issue:`33721`.) -Added :meth:`pathlib.Path.link_to()` which creates a hard link pointing +Added :meth:`!pathlib.Path.link_to()` which creates a hard link pointing to a path. (Contributed by Joannah Nanjekye in :issue:`26978`) Note that ``link_to`` was deprecated in 3.10 and removed in 3.12 in diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index cb2482ee48d7fa..f7ad4372325ccb 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -585,7 +585,7 @@ queue. nntplib ------- -:class:`~!nntplib.NNTP` and :class:`~!nntplib.NNTP_SSL` now raise a :class:`ValueError` +:class:`!NNTP` and :class:`!NNTP_SSL` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) @@ -931,7 +931,7 @@ Deprecated * Passing ``None`` as the first argument to the :func:`shlex.split` function has been deprecated. (Contributed by Zackery Spytz in :issue:`33262`.) -* :func:`smtpd.MailmanProxy` is now deprecated as it is unusable without +* :func:`!smtpd.MailmanProxy` is now deprecated as it is unusable without an external module, ``mailman``. (Contributed by Samuel Colvin in :issue:`35800`.) * The :mod:`!lib2to3` module now emits a :exc:`PendingDeprecationWarning`. diff --git a/Include/boolobject.h b/Include/boolobject.h index 976fa35201d035..19aef5b1b87c6a 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -7,7 +7,7 @@ extern "C" { #endif -PyAPI_DATA(PyTypeObject) PyBool_Type; +// PyBool_Type is declared by object.h #define PyBool_Check(x) Py_IS_TYPE((x), &PyBool_Type) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 762e8a3b86ee1e..d6482f4bc689a1 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -39,6 +39,10 @@ typedef struct _Py_Identifier { // Index in PyInterpreterState.unicode.ids.array. It is process-wide // unique and must be initialized to -1. Py_ssize_t index; + // Hidden PyMutex struct for non free-threaded build. + struct { + uint8_t v; + } mutex; } _Py_Identifier; #ifndef Py_BUILD_CORE diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index adc2c1fc442280..d521eac79d1b97 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -32,7 +32,7 @@ typedef struct { typedef struct _PyExecutorObject { PyObject_VAR_HEAD /* WARNING: execute consumes a reference to self. This is necessary to allow executors to tail call into each other. */ - struct _PyInterpreterFrame *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); + _Py_CODEUNIT *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ /* Data needed by the executor goes here, but is opaque to the VM */ } _PyExecutorObject; diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index 6f7113daa5fe4d..396c208e1b106a 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -21,7 +21,9 @@ /* Helpers for hash functions */ PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); -PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*); + +// Kept for backward compatibility +#define _Py_HashPointer Py_HashPointer /* hash function definition */ @@ -33,3 +35,5 @@ typedef struct { } PyHash_FuncDef; PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); + +PyAPI_FUNC(Py_hash_t) Py_HashPointer(const void *ptr); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 56172d231c44f4..ed7dd829d4b6f0 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -223,9 +223,11 @@ struct _ts { // layout, optimization, and WASI runtime. Wasmtime can handle about 700 // recursions, sometimes less. 500 is a more conservative limit. # define Py_C_RECURSION_LIMIT 500 +#elif defined(__s390x__) +# define Py_C_RECURSION_LIMIT 1200 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 1500 +# define Py_C_RECURSION_LIMIT 8000 #endif diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 294bf1505f0115..ba67eefef3e37a 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -114,6 +114,7 @@ typedef struct _optimization_stats { uint64_t trace_too_short; uint64_t inner_loop; uint64_t recursive_call; + uint64_t low_confidence; UOpStats opcode[512]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h index 9fd7cc0cb43931..a3ac07f538a94f 100644 --- a/Include/cpython/sysmodule.h +++ b/Include/cpython/sysmodule.h @@ -4,10 +4,6 @@ typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); -PyAPI_FUNC(int) PySys_Audit( - const char *event, - const char *format, - ...); PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); typedef struct { diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index f076bc6a40f977..cb6e211de5bb63 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -85,6 +85,7 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap void _mi_thread_done(mi_heap_t* heap); void _mi_thread_data_collect(void); +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); // os.c void _mi_os_init(void); // called from process init @@ -170,6 +171,7 @@ size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats // "heap.c" +void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id); void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); void _mi_heap_set_default_direct(mi_heap_t* heap); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index 6ffd30aca7b11b..f1b1786264803b 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -16,8 +16,6 @@ extern "C" { struct ast_state { _PyOnceFlag once; int finalized; - int recursion_depth; - int recursion_limit; PyObject *AST_type; PyObject *Add_singleton; PyObject *Add_type; diff --git a/Include/internal/pycore_atexit.h b/Include/internal/pycore_atexit.h index 3966df70e2616f..4dcda8f517c787 100644 --- a/Include/internal/pycore_atexit.h +++ b/Include/internal/pycore_atexit.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_ATEXIT_H #define Py_INTERNAL_ATEXIT_H + +#include "pycore_lock.h" // PyMutex + #ifdef __cplusplus extern "C" { #endif @@ -15,7 +18,7 @@ extern "C" { typedef void (*atexit_callbackfunc)(void); struct _atexit_runtime_state { - PyThread_type_lock mutex; + PyMutex mutex; #define NEXITFUNCS 32 atexit_callbackfunc callbacks[NEXITFUNCS]; int ncallbacks; diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index c372b7224fb047..a357bfa3a26064 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -41,8 +41,7 @@ PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); #endif extern void _Py_FinishPendingCalls(PyThreadState *tstate); -extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); -extern void _PyEval_FiniState(struct _ceval_state *ceval); +extern void _PyEval_InitState(PyInterpreterState *); extern void _PyEval_SignalReceived(PyInterpreterState *interp); // bitwise flags: @@ -101,6 +100,7 @@ extern int _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *); extern void _PyPerfTrampoline_GetCallbacks(_PyPerf_Callbacks *); extern int _PyPerfTrampoline_Init(int activate); extern int _PyPerfTrampoline_Fini(void); +extern void _PyPerfTrampoline_FreeArenas(void); extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -124,7 +124,7 @@ _PyEval_Vector(PyThreadState *tstate, PyObject *kwnames); extern int _PyEval_ThreadsInitialized(void); -extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil); +extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_AcquireLock(PyThreadState *tstate); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 072bbcda0c3c82..28738980eb49be 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex #include "pycore_gil.h" // struct _gil_runtime_state @@ -15,7 +16,7 @@ typedef int (*_Py_pending_call_func)(void *); struct _pending_calls { int busy; - PyThread_type_lock lock; + PyMutex mutex; /* Request for running pending calls. */ int32_t calls_to_do; #define NPENDINGCALLS 32 diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index eaf84a9c94fc9b..73df6c3568ffe0 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -394,27 +394,29 @@ write_varint(uint8_t *ptr, unsigned int val) val >>= 6; written++; } - *ptr = val; + *ptr = (uint8_t)val; return written; } static inline int write_signed_varint(uint8_t *ptr, int val) { + unsigned int uval; if (val < 0) { - val = ((-val)<<1) | 1; + // (unsigned int)(-val) has an undefined behavior for INT_MIN + uval = ((0 - (unsigned int)val) << 1) | 1; } else { - val = val << 1; + uval = (unsigned int)val << 1; } - return write_varint(ptr, val); + return write_varint(ptr, uval); } static inline int write_location_entry_start(uint8_t *ptr, int code, int length) { assert((code & 15) == code); - *ptr = 128 | (code << 3) | (length - 1); + *ptr = 128 | (uint8_t)(code << 3) | (uint8_t)(length - 1); return 1; } @@ -454,9 +456,9 @@ write_location_entry_start(uint8_t *ptr, int code, int length) static inline uint16_t -adaptive_counter_bits(int value, int backoff) { - return (value << ADAPTIVE_BACKOFF_BITS) | - (backoff & ((1< MAX_BACKOFF_VALUE) { backoff = MAX_BACKOFF_VALUE; } - unsigned int value = (1 << backoff) - 1; + uint16_t value = (uint16_t)(1 << backoff) - 1; return adaptive_counter_bits(value, backoff); } diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index ec9dac96292f35..d6e297a7e8e6db 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -8,8 +8,16 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex #include "pycore_pyerrors.h" +/**************/ +/* exceptions */ +/**************/ + +PyAPI_DATA(PyObject *) PyExc_InterpreterError; +PyAPI_DATA(PyObject *) PyExc_InterpreterNotFoundError; + /***************************/ /* cross-interpreter calls */ @@ -128,7 +136,7 @@ struct _xidregitem { struct _xidregistry { int global; /* builtin types or heap types */ int initialized; - PyThread_type_lock mutex; + PyMutex mutex; struct _xidregitem *head; }; @@ -159,6 +167,9 @@ struct _xi_state { extern PyStatus _PyXI_Init(PyInterpreterState *interp); extern void _PyXI_Fini(PyInterpreterState *interp); +extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp); +extern void _PyXI_FiniTypes(PyInterpreterState *interp); + /***************************/ /* short-term data sharing */ @@ -177,6 +188,7 @@ typedef struct _excinfo { const char *module; } type; const char *msg; + const char *errdisplay; } _PyXI_excinfo; diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index ac62a4d300720a..c5cfdf4ce8f823 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -35,6 +35,9 @@ struct _dtoa_state { /* The size of the Bigint freelist */ #define Bigint_Kmax 7 +/* The size of the cached powers of 5 array */ +#define Bigint_Pow5size 8 + #ifndef PRIVATE_MEM #define PRIVATE_MEM 2304 #endif @@ -42,9 +45,10 @@ struct _dtoa_state { ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) struct _dtoa_state { - /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ + // p5s is an array of powers of 5 of the form: + // 5**(2**(i+2)) for 0 <= i < Bigint_Pow5size + struct Bigint *p5s[Bigint_Pow5size]; // XXX This should be freed during runtime fini. - struct Bigint *p5s; struct Bigint *freelist[Bigint_Kmax+1]; double preallocated[Bigint_PREALLOC_SIZE]; double *preallocated_next; @@ -57,9 +61,6 @@ struct _dtoa_state { #endif // !Py_USING_MEMORY_DEBUGGER -/* These functions are used by modules compiled as C extension like math: - they must be exported. */ - extern double _Py_dg_strtod(const char *str, char **ptr); extern char* _Py_dg_dtoa(double d, int mode, int ndigits, int *decpt, int *sign, char **rve); @@ -67,6 +68,11 @@ extern void _Py_dg_freedtoa(char *s); #endif // _PY_SHORT_FLOAT_REPR == 1 + +extern PyStatus _PyDtoa_Init(PyInterpreterState *interp); +extern void _PyDtoa_Fini(PyInterpreterState *interp); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 2f89da2c6ecd91..5c55282fa39e6f 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -320,6 +320,10 @@ PyAPI_FUNC(char*) _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject extern int _PyFile_Flush(PyObject *); +#ifndef MS_WINDOWS +extern int _Py_GetTicksPerSecond(long *ticks_per_second); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 117e46bb86285d..c84f87a831bf38 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -9,6 +9,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex #include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_time.h" // _PyTime_t @@ -47,7 +48,7 @@ struct _import_runtime_state { Py_ssize_t last_module_index; struct { /* A lock to guard the cache. */ - PyThread_type_lock mutex; + PyMutex mutex; /* The actual cache of (filename, name, PyModuleDef) for modules. Only legacy (single-phase init) extension modules are added and only if they support multiple initialization (m_size >- 0) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 498db8becf114c..04d7a6a615e370 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -29,6 +29,7 @@ extern "C" { #include "pycore_list.h" // struct _Py_list_state #include "pycore_object_state.h" // struct _py_object_state #include "pycore_obmalloc.h" // struct _obmalloc_state +#include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state @@ -210,8 +211,8 @@ struct _is { struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; - /* the initial PyInterpreterState.threads.head */ - PyThreadState _initial_thread; + /* the initial PyInterpreterState.threads.head */ + _PyThreadStateImpl _initial_thread; Py_ssize_t _interactive_src_count; }; @@ -249,9 +250,9 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst // Export for the _xxinterpchannels module. PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); -extern int _PyInterpreterState_IDInitref(PyInterpreterState *); -extern int _PyInterpreterState_IDIncref(PyInterpreterState *); -extern void _PyInterpreterState_IDDecref(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); +PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index f135cbbc3754fb..18a8896d97a548 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -92,6 +92,13 @@ PyMutex_IsLocked(PyMutex *m) return (_Py_atomic_load_uint8(&m->v) & _Py_LOCKED) != 0; } +// Re-initializes the mutex after a fork to the unlocked state. +static inline void +_PyMutex_at_fork_reinit(PyMutex *m) +{ + memset(m, 0, sizeof(*m)); +} + typedef enum _PyLockFlags { // Do not detach/release the GIL when waiting on the lock. _Py_LOCK_DONT_DETACH = 0, @@ -108,6 +115,16 @@ typedef enum _PyLockFlags { extern PyLockStatus _PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout_ns, _PyLockFlags flags); +// Lock a mutex with aditional options. See _PyLockFlags for details. +static inline void +PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) +{ + uint8_t expected = _Py_UNLOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_LOCKED)) { + _PyMutex_LockTimed(m, -1, flags); + } +} + // Unlock a mutex, returns 0 if the mutex is not locked (used for improved // error messages). extern int _PyMutex_TryUnlock(PyMutex *m); @@ -196,6 +213,45 @@ _PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) return _PyOnceFlag_CallOnceSlow(flag, fn, arg); } +// A readers-writer (RW) lock. The lock supports multiple concurrent readers or +// a single writer. The lock is write-preferring: if a writer is waiting while +// the lock is read-locked then, new readers will be blocked. This avoids +// starvation of writers. +// +// In C++, the equivalent synchronization primitive is std::shared_mutex +// with shared ("read") and exclusive ("write") locking. +// +// The two least significant bits are used to indicate if the lock is +// write-locked and if there are parked threads (either readers or writers) +// waiting to acquire the lock. The remaining bits are used to indicate the +// number of readers holding the lock. +// +// 0b000..00000: unlocked +// 0bnnn..nnn00: nnn..nnn readers holding the lock +// 0bnnn..nnn10: nnn..nnn readers holding the lock and a writer is waiting +// 0b00000..010: unlocked with awoken writer about to acquire lock +// 0b00000..001: write-locked +// 0b00000..011: write-locked and readers or other writers are waiting +// +// Note that reader_count must be zero if the lock is held by a writer, and +// vice versa. The lock can only be held by readers or a writer, but not both. +// +// The design is optimized for simplicity of the implementation. The lock is +// not fair: if fairness is desired, use an additional PyMutex to serialize +// writers. The lock is also not reentrant. +typedef struct { + uintptr_t bits; +} _PyRWMutex; + +// Read lock (i.e., shared lock) +PyAPI_FUNC(void) _PyRWMutex_RLock(_PyRWMutex *rwmutex); +PyAPI_FUNC(void) _PyRWMutex_RUnlock(_PyRWMutex *rwmutex); + +// Write lock (i.e., exclusive lock) +PyAPI_FUNC(void) _PyRWMutex_Lock(_PyRWMutex *rwmutex); +PyAPI_FUNC(void) _PyRWMutex_Unlock(_PyRWMutex *rwmutex); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index c29dc82a42762a..adebb559dae658 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -9,11 +9,37 @@ # error "pycore_mimalloc.h must be included before mimalloc.h" #endif +typedef enum { + _Py_MIMALLOC_HEAP_MEM = 0, // PyMem_Malloc() and friends + _Py_MIMALLOC_HEAP_OBJECT = 1, // non-GC objects + _Py_MIMALLOC_HEAP_GC = 2, // GC objects without pre-header + _Py_MIMALLOC_HEAP_GC_PRE = 3, // GC objects with pre-header + _Py_MIMALLOC_HEAP_COUNT +} _Py_mimalloc_heap_id; + #include "pycore_pymem.h" + +#ifdef WITH_MIMALLOC #define MI_DEBUG_UNINIT PYMEM_CLEANBYTE #define MI_DEBUG_FREED PYMEM_DEADBYTE #define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE +#ifdef Py_DEBUG +# define MI_DEBUG 1 +#else +# define MI_DEBUG 0 +#endif #include "mimalloc.h" +#include "mimalloc/types.h" +#include "mimalloc/internal.h" +#endif + +#ifdef Py_GIL_DISABLED +struct _mimalloc_thread_state { + mi_heap_t *current_object_heap; + mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT]; + mi_tld_t tld; +}; +#endif #endif // Py_INTERNAL_MIMALLOC_H diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index b0dbf53d4e3d15..17572dba65487d 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -665,7 +665,9 @@ struct _obmalloc_global_state { struct _obmalloc_state { struct _obmalloc_pools pools; struct _obmalloc_mgmt mgmt; +#if WITH_PYMALLOC_RADIX_TREE struct _obmalloc_usage usage; +#endif }; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4e45725d393479..7d39e4bc03099c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1,13 +1,20 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/opcode_metadata_generator.py // from: // Python/bytecodes.c // Do not edit! +#ifndef Py_CORE_OPCODE_METADATA_H +#define Py_CORE_OPCODE_METADATA_H +#ifdef __cplusplus +extern "C" { +#endif + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif #include // bool +#include "opcode_ids.h" #define IS_PSEUDO_INSTR(OP) ( \ @@ -25,498 +32,318 @@ ((OP) == POP_BLOCK) || \ 0) -#define _EXIT_TRACE 300 -#define _SET_IP 301 -#define _SPECIALIZE_TO_BOOL 302 -#define _TO_BOOL 303 -#define _GUARD_BOTH_INT 304 -#define _BINARY_OP_MULTIPLY_INT 305 -#define _BINARY_OP_ADD_INT 306 -#define _BINARY_OP_SUBTRACT_INT 307 -#define _GUARD_BOTH_FLOAT 308 -#define _BINARY_OP_MULTIPLY_FLOAT 309 -#define _BINARY_OP_ADD_FLOAT 310 -#define _BINARY_OP_SUBTRACT_FLOAT 311 -#define _GUARD_BOTH_UNICODE 312 -#define _BINARY_OP_ADD_UNICODE 313 -#define _BINARY_OP_INPLACE_ADD_UNICODE 314 -#define _SPECIALIZE_BINARY_SUBSCR 315 -#define _BINARY_SUBSCR 316 -#define _SPECIALIZE_STORE_SUBSCR 317 -#define _STORE_SUBSCR 318 -#define _POP_FRAME 319 -#define _SPECIALIZE_SEND 320 -#define _SEND 321 -#define _SPECIALIZE_UNPACK_SEQUENCE 322 -#define _UNPACK_SEQUENCE 323 -#define _SPECIALIZE_STORE_ATTR 324 -#define _STORE_ATTR 325 -#define _SPECIALIZE_LOAD_GLOBAL 326 -#define _LOAD_GLOBAL 327 -#define _GUARD_GLOBALS_VERSION 328 -#define _GUARD_BUILTINS_VERSION 329 -#define _LOAD_GLOBAL_MODULE 330 -#define _LOAD_GLOBAL_BUILTINS 331 -#define _SPECIALIZE_LOAD_SUPER_ATTR 332 -#define _LOAD_SUPER_ATTR 333 -#define _SPECIALIZE_LOAD_ATTR 334 -#define _LOAD_ATTR 335 -#define _GUARD_TYPE_VERSION 336 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337 -#define _LOAD_ATTR_INSTANCE_VALUE 338 -#define _CHECK_ATTR_MODULE 339 -#define _LOAD_ATTR_MODULE 340 -#define _CHECK_ATTR_WITH_HINT 341 -#define _LOAD_ATTR_WITH_HINT 342 -#define _LOAD_ATTR_SLOT 343 -#define _CHECK_ATTR_CLASS 344 -#define _LOAD_ATTR_CLASS 345 -#define _GUARD_DORV_VALUES 346 -#define _STORE_ATTR_INSTANCE_VALUE 347 -#define _STORE_ATTR_SLOT 348 -#define _SPECIALIZE_COMPARE_OP 349 -#define _COMPARE_OP 350 -#define _POP_JUMP_IF_FALSE 351 -#define _POP_JUMP_IF_TRUE 352 -#define _IS_NONE 353 -#define _SPECIALIZE_FOR_ITER 354 -#define _FOR_ITER 355 -#define _FOR_ITER_TIER_TWO 356 -#define _ITER_CHECK_LIST 357 -#define _ITER_JUMP_LIST 358 -#define _GUARD_NOT_EXHAUSTED_LIST 359 -#define _ITER_NEXT_LIST 360 -#define _ITER_CHECK_TUPLE 361 -#define _ITER_JUMP_TUPLE 362 -#define _GUARD_NOT_EXHAUSTED_TUPLE 363 -#define _ITER_NEXT_TUPLE 364 -#define _ITER_CHECK_RANGE 365 -#define _ITER_JUMP_RANGE 366 -#define _GUARD_NOT_EXHAUSTED_RANGE 367 -#define _ITER_NEXT_RANGE 368 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 369 -#define _GUARD_KEYS_VERSION 370 -#define _LOAD_ATTR_METHOD_WITH_VALUES 371 -#define _LOAD_ATTR_METHOD_NO_DICT 372 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 373 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 374 -#define _CHECK_ATTR_METHOD_LAZY_DICT 375 -#define _LOAD_ATTR_METHOD_LAZY_DICT 376 -#define _SPECIALIZE_CALL 377 -#define _CALL 378 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 379 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 380 -#define _CHECK_PEP_523 381 -#define _CHECK_FUNCTION_EXACT_ARGS 382 -#define _CHECK_STACK_SPACE 383 -#define _INIT_CALL_PY_EXACT_ARGS 384 -#define _PUSH_FRAME 385 -#define _SPECIALIZE_BINARY_OP 386 -#define _BINARY_OP 387 -#define _GUARD_IS_TRUE_POP 388 -#define _GUARD_IS_FALSE_POP 389 -#define _GUARD_IS_NONE_POP 390 -#define _GUARD_IS_NOT_NONE_POP 391 -#define _JUMP_TO_TOP 392 -#define _SAVE_RETURN_OFFSET 393 -#define _INSERT 394 -#define _CHECK_VALIDITY 395 - -extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); +#include "pycore_uop_ids.h" +extern int _PyOpcode_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA -int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { +int _PyOpcode_num_popped(int opcode, int oparg) { switch(opcode) { - case NOP: - return 0; - case RESUME: - return 0; - case RESUME_CHECK: - return 0; - case INSTRUMENTED_RESUME: - return 0; - case LOAD_CLOSURE: - return 0; - case LOAD_FAST_CHECK: - return 0; - case LOAD_FAST: - return 0; - case LOAD_FAST_AND_CLEAR: - return 0; - case LOAD_FAST_LOAD_FAST: - return 0; - case LOAD_CONST: - return 0; - case STORE_FAST: - return 1; - case STORE_FAST_MAYBE_NULL: - return 1; - case STORE_FAST_LOAD_FAST: - return 1; - case STORE_FAST_STORE_FAST: - return 2; - case POP_TOP: - return 1; - case PUSH_NULL: - return 0; - case END_FOR: - return 2; - case INSTRUMENTED_END_FOR: - return 2; - case END_SEND: - return 2; - case INSTRUMENTED_END_SEND: - return 2; - case UNARY_NEGATIVE: - return 1; - case UNARY_NOT: - return 1; - case _SPECIALIZE_TO_BOOL: - return 1; - case _TO_BOOL: - return 1; - case TO_BOOL: - return 1; - case TO_BOOL_BOOL: - return 1; - case TO_BOOL_INT: - return 1; - case TO_BOOL_LIST: - return 1; - case TO_BOOL_NONE: - return 1; - case TO_BOOL_STR: - return 1; - case TO_BOOL_ALWAYS_TRUE: + case BEFORE_ASYNC_WITH: return 1; - case UNARY_INVERT: + case BEFORE_WITH: return 1; - case _GUARD_BOTH_INT: - return 2; - case _BINARY_OP_MULTIPLY_INT: - return 2; - case _BINARY_OP_ADD_INT: - return 2; - case _BINARY_OP_SUBTRACT_INT: + case BINARY_OP: return 2; - case BINARY_OP_MULTIPLY_INT: + case BINARY_OP_ADD_FLOAT: return 2; case BINARY_OP_ADD_INT: return 2; - case BINARY_OP_SUBTRACT_INT: - return 2; - case _GUARD_BOTH_FLOAT: - return 2; - case _BINARY_OP_MULTIPLY_FLOAT: - return 2; - case _BINARY_OP_ADD_FLOAT: + case BINARY_OP_ADD_UNICODE: return 2; - case _BINARY_OP_SUBTRACT_FLOAT: + case BINARY_OP_INPLACE_ADD_UNICODE: return 2; case BINARY_OP_MULTIPLY_FLOAT: return 2; - case BINARY_OP_ADD_FLOAT: + case BINARY_OP_MULTIPLY_INT: return 2; case BINARY_OP_SUBTRACT_FLOAT: return 2; - case _GUARD_BOTH_UNICODE: - return 2; - case _BINARY_OP_ADD_UNICODE: - return 2; - case BINARY_OP_ADD_UNICODE: - return 2; - case _BINARY_OP_INPLACE_ADD_UNICODE: - return 2; - case BINARY_OP_INPLACE_ADD_UNICODE: + case BINARY_OP_SUBTRACT_INT: return 2; - case _SPECIALIZE_BINARY_SUBSCR: + case BINARY_SLICE: + return 3; + case BINARY_SUBSCR: return 2; - case _BINARY_SUBSCR: + case BINARY_SUBSCR_DICT: return 2; - case BINARY_SUBSCR: + case BINARY_SUBSCR_GETITEM: return 2; - case BINARY_SLICE: - return 3; - case STORE_SLICE: - return 4; case BINARY_SUBSCR_LIST_INT: return 2; case BINARY_SUBSCR_STR_INT: return 2; case BINARY_SUBSCR_TUPLE_INT: return 2; - case BINARY_SUBSCR_DICT: + case BUILD_CONST_KEY_MAP: + return 1 + oparg; + case BUILD_LIST: + return oparg; + case BUILD_MAP: + return oparg*2; + case BUILD_SET: + return oparg; + case BUILD_SLICE: + return 2 + ((oparg == 3) ? 1 : 0); + case BUILD_STRING: + return oparg; + case BUILD_TUPLE: + return oparg; + case CACHE: + return 0; + case CALL: + return 2 + oparg; + case CALL_ALLOC_AND_ENTER_INIT: + return 2 + oparg; + case CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case CALL_BUILTIN_CLASS: + return 2 + oparg; + case CALL_BUILTIN_FAST: + return 2 + oparg; + case CALL_BUILTIN_FAST_WITH_KEYWORDS: + return 2 + oparg; + case CALL_BUILTIN_O: + return 2 + oparg; + case CALL_FUNCTION_EX: + return 3 + (oparg & 1); + case CALL_INTRINSIC_1: + return 1; + case CALL_INTRINSIC_2: return 2; - case BINARY_SUBSCR_GETITEM: + case CALL_ISINSTANCE: + return 2 + oparg; + case CALL_KW: + return 3 + oparg; + case CALL_LEN: + return 2 + oparg; + case CALL_LIST_APPEND: + return 2 + oparg; + case CALL_METHOD_DESCRIPTOR_FAST: + return 2 + oparg; + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 2 + oparg; + case CALL_METHOD_DESCRIPTOR_NOARGS: + return 2 + oparg; + case CALL_METHOD_DESCRIPTOR_O: + return 2 + oparg; + case CALL_PY_EXACT_ARGS: + return 2 + oparg; + case CALL_PY_WITH_DEFAULTS: + return 2 + oparg; + case CALL_STR_1: + return 2 + oparg; + case CALL_TUPLE_1: + return 2 + oparg; + case CALL_TYPE_1: + return 2 + oparg; + case CHECK_EG_MATCH: return 2; - case LIST_APPEND: - return (oparg-1) + 2; - case SET_ADD: - return (oparg-1) + 2; - case _SPECIALIZE_STORE_SUBSCR: + case CHECK_EXC_MATCH: return 2; - case _STORE_SUBSCR: - return 3; - case STORE_SUBSCR: - return 3; - case STORE_SUBSCR_LIST_INT: - return 3; - case STORE_SUBSCR_DICT: + case CLEANUP_THROW: return 3; - case DELETE_SUBSCR: + case COMPARE_OP: return 2; - case CALL_INTRINSIC_1: - return 1; - case CALL_INTRINSIC_2: + case COMPARE_OP_FLOAT: return 2; - case RAISE_VARARGS: - return oparg; - case INTERPRETER_EXIT: - return 1; - case _POP_FRAME: - return 1; - case RETURN_VALUE: + case COMPARE_OP_INT: + return 2; + case COMPARE_OP_STR: + return 2; + case CONTAINS_OP: + return 2; + case CONVERT_VALUE: return 1; - case INSTRUMENTED_RETURN_VALUE: + case COPY: + return 1 + (oparg-1); + case COPY_FREE_VARS: + return 0; + case DELETE_ATTR: return 1; - case RETURN_CONST: + case DELETE_DEREF: return 0; - case INSTRUMENTED_RETURN_CONST: + case DELETE_FAST: return 0; - case GET_AITER: - return 1; - case GET_ANEXT: - return 1; - case GET_AWAITABLE: - return 1; - case _SPECIALIZE_SEND: + case DELETE_GLOBAL: + return 0; + case DELETE_NAME: + return 0; + case DELETE_SUBSCR: return 2; - case _SEND: + case DICT_MERGE: + return 5 + (oparg - 1); + case DICT_UPDATE: + return 2 + (oparg - 1); + case END_ASYNC_FOR: return 2; - case SEND: + case END_FOR: return 2; - case SEND_GEN: + case END_SEND: return 2; - case INSTRUMENTED_YIELD_VALUE: - return 1; - case YIELD_VALUE: + case ENTER_EXECUTOR: + return 0; + case EXIT_INIT_CHECK: return 1; - case POP_EXCEPT: + case EXTENDED_ARG: + return 0; + case FORMAT_SIMPLE: return 1; - case RERAISE: - return oparg + 1; - case END_ASYNC_FOR: + case FORMAT_WITH_SPEC: return 2; - case CLEANUP_THROW: - return 3; - case LOAD_ASSERTION_ERROR: - return 0; - case LOAD_BUILD_CLASS: - return 0; - case STORE_NAME: + case FOR_ITER: return 1; - case DELETE_NAME: - return 0; - case _SPECIALIZE_UNPACK_SEQUENCE: + case FOR_ITER_GEN: return 1; - case _UNPACK_SEQUENCE: + case FOR_ITER_LIST: return 1; - case UNPACK_SEQUENCE: + case FOR_ITER_RANGE: return 1; - case UNPACK_SEQUENCE_TWO_TUPLE: + case FOR_ITER_TUPLE: return 1; - case UNPACK_SEQUENCE_TUPLE: + case GET_AITER: return 1; - case UNPACK_SEQUENCE_LIST: + case GET_ANEXT: return 1; - case UNPACK_EX: + case GET_AWAITABLE: return 1; - case _SPECIALIZE_STORE_ATTR: + case GET_ITER: return 1; - case _STORE_ATTR: - return 2; - case STORE_ATTR: - return 2; - case DELETE_ATTR: + case GET_LEN: return 1; - case STORE_GLOBAL: + case GET_YIELD_FROM_ITER: return 1; - case DELETE_GLOBAL: - return 0; - case LOAD_LOCALS: - return 0; - case LOAD_FROM_DICT_OR_GLOBALS: + case IMPORT_FROM: return 1; - case LOAD_NAME: - return 0; - case _SPECIALIZE_LOAD_GLOBAL: + case IMPORT_NAME: + return 2; + case INSTRUMENTED_CALL: return 0; - case _LOAD_GLOBAL: + case INSTRUMENTED_CALL_FUNCTION_EX: return 0; - case LOAD_GLOBAL: + case INSTRUMENTED_CALL_KW: return 0; - case _GUARD_GLOBALS_VERSION: + case INSTRUMENTED_END_FOR: + return 2; + case INSTRUMENTED_END_SEND: + return 2; + case INSTRUMENTED_FOR_ITER: return 0; - case _GUARD_BUILTINS_VERSION: + case INSTRUMENTED_INSTRUCTION: return 0; - case _LOAD_GLOBAL_MODULE: + case INSTRUMENTED_JUMP_BACKWARD: return 0; - case _LOAD_GLOBAL_BUILTINS: + case INSTRUMENTED_JUMP_FORWARD: return 0; - case LOAD_GLOBAL_MODULE: + case INSTRUMENTED_LOAD_SUPER_ATTR: + return 3; + case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; - case LOAD_GLOBAL_BUILTIN: + case INSTRUMENTED_POP_JUMP_IF_NONE: return 0; - case DELETE_FAST: + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: return 0; - case MAKE_CELL: + case INSTRUMENTED_POP_JUMP_IF_TRUE: return 0; - case DELETE_DEREF: + case INSTRUMENTED_RESUME: return 0; - case LOAD_FROM_DICT_OR_DEREF: - return 1; - case LOAD_DEREF: + case INSTRUMENTED_RETURN_CONST: return 0; - case STORE_DEREF: + case INSTRUMENTED_RETURN_VALUE: return 1; - case COPY_FREE_VARS: + case INSTRUMENTED_YIELD_VALUE: + return 1; + case INTERPRETER_EXIT: + return 1; + case IS_OP: + return 2; + case JUMP_BACKWARD: return 0; - case BUILD_STRING: - return oparg; - case BUILD_TUPLE: - return oparg; - case BUILD_LIST: - return oparg; + case JUMP_BACKWARD_NO_INTERRUPT: + return 0; + case JUMP_FORWARD: + return 0; + case LIST_APPEND: + return 2 + (oparg-1); case LIST_EXTEND: - return (oparg-1) + 2; - case SET_UPDATE: - return (oparg-1) + 2; - case BUILD_SET: - return oparg; - case BUILD_MAP: - return oparg*2; - case SETUP_ANNOTATIONS: + return 2 + (oparg-1); + case LOAD_ASSERTION_ERROR: return 0; - case BUILD_CONST_KEY_MAP: - return oparg + 1; - case DICT_UPDATE: - return (oparg - 1) + 2; - case DICT_MERGE: - return (oparg - 1) + 5; - case MAP_ADD: - return (oparg - 1) + 3; - case INSTRUMENTED_LOAD_SUPER_ATTR: - return 3; - case _SPECIALIZE_LOAD_SUPER_ATTR: - return 3; - case _LOAD_SUPER_ATTR: - return 3; - case LOAD_SUPER_ATTR: - return 3; - case LOAD_SUPER_METHOD: - return 3; - case LOAD_ZERO_SUPER_METHOD: - return 3; - case LOAD_ZERO_SUPER_ATTR: - return 3; - case LOAD_SUPER_ATTR_ATTR: - return 3; - case LOAD_SUPER_ATTR_METHOD: - return 3; - case _SPECIALIZE_LOAD_ATTR: - return 1; - case _LOAD_ATTR: - return 1; case LOAD_ATTR: return 1; - case LOAD_METHOD: - return 1; - case _GUARD_TYPE_VERSION: - return 1; - case _CHECK_MANAGED_OBJECT_HAS_VALUES: + case LOAD_ATTR_CLASS: return 1; - case _LOAD_ATTR_INSTANCE_VALUE: + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: return 1; case LOAD_ATTR_INSTANCE_VALUE: return 1; - case _CHECK_ATTR_MODULE: + case LOAD_ATTR_METHOD_LAZY_DICT: + return 1; + case LOAD_ATTR_METHOD_NO_DICT: return 1; - case _LOAD_ATTR_MODULE: + case LOAD_ATTR_METHOD_WITH_VALUES: return 1; case LOAD_ATTR_MODULE: return 1; - case _CHECK_ATTR_WITH_HINT: - return 1; - case _LOAD_ATTR_WITH_HINT: + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: return 1; - case LOAD_ATTR_WITH_HINT: + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; - case _LOAD_ATTR_SLOT: + case LOAD_ATTR_PROPERTY: return 1; case LOAD_ATTR_SLOT: return 1; - case _CHECK_ATTR_CLASS: - return 1; - case _LOAD_ATTR_CLASS: - return 1; - case LOAD_ATTR_CLASS: - return 1; - case LOAD_ATTR_PROPERTY: - return 1; - case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: + case LOAD_ATTR_WITH_HINT: return 1; - case _GUARD_DORV_VALUES: + case LOAD_BUILD_CLASS: + return 0; + case LOAD_CONST: + return 0; + case LOAD_DEREF: + return 0; + case LOAD_FAST: + return 0; + case LOAD_FAST_AND_CLEAR: + return 0; + case LOAD_FAST_CHECK: + return 0; + case LOAD_FAST_LOAD_FAST: + return 0; + case LOAD_FROM_DICT_OR_DEREF: return 1; - case _STORE_ATTR_INSTANCE_VALUE: - return 2; - case STORE_ATTR_INSTANCE_VALUE: - return 2; - case STORE_ATTR_WITH_HINT: - return 2; - case _STORE_ATTR_SLOT: - return 2; - case STORE_ATTR_SLOT: - return 2; - case _SPECIALIZE_COMPARE_OP: - return 2; - case _COMPARE_OP: - return 2; - case COMPARE_OP: - return 2; - case COMPARE_OP_FLOAT: - return 2; - case COMPARE_OP_INT: - return 2; - case COMPARE_OP_STR: - return 2; - case IS_OP: - return 2; - case CONTAINS_OP: - return 2; - case CHECK_EG_MATCH: - return 2; - case CHECK_EXC_MATCH: - return 2; - case IMPORT_NAME: - return 2; - case IMPORT_FROM: + case LOAD_FROM_DICT_OR_GLOBALS: return 1; - case JUMP_FORWARD: + case LOAD_GLOBAL: return 0; - case JUMP_BACKWARD: + case LOAD_GLOBAL_BUILTIN: return 0; - case JUMP: + case LOAD_GLOBAL_MODULE: return 0; - case JUMP_NO_INTERRUPT: + case LOAD_LOCALS: return 0; - case ENTER_EXECUTOR: + case LOAD_NAME: + return 0; + case LOAD_SUPER_ATTR: + return 3; + case LOAD_SUPER_ATTR_ATTR: + return 3; + case LOAD_SUPER_ATTR_METHOD: + return 3; + case MAKE_CELL: return 0; - case _POP_JUMP_IF_FALSE: + case MAKE_FUNCTION: return 1; - case _POP_JUMP_IF_TRUE: + case MAP_ADD: + return 3 + (oparg - 1); + case MATCH_CLASS: + return 3; + case MATCH_KEYS: + return 2; + case MATCH_MAPPING: return 1; - case _IS_NONE: + case MATCH_SEQUENCE: return 1; - case POP_JUMP_IF_TRUE: + case NOP: + return 0; + case POP_EXCEPT: return 1; case POP_JUMP_IF_FALSE: return 1; @@ -524,901 +351,552 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case POP_JUMP_IF_NOT_NONE: return 1; - case JUMP_BACKWARD_NO_INTERRUPT: - return 0; - case GET_LEN: - return 1; - case MATCH_CLASS: - return 3; - case MATCH_MAPPING: - return 1; - case MATCH_SEQUENCE: - return 1; - case MATCH_KEYS: - return 2; - case GET_ITER: + case POP_JUMP_IF_TRUE: return 1; - case GET_YIELD_FROM_ITER: + case POP_TOP: return 1; - case _SPECIALIZE_FOR_ITER: + case PUSH_EXC_INFO: return 1; - case _FOR_ITER: + case PUSH_NULL: + return 0; + case RAISE_VARARGS: + return oparg; + case RERAISE: + return 1 + oparg; + case RESERVED: + return 0; + case RESUME: + return 0; + case RESUME_CHECK: + return 0; + case RETURN_CONST: + return 0; + case RETURN_GENERATOR: + return 0; + case RETURN_VALUE: return 1; - case _FOR_ITER_TIER_TWO: + case SEND: + return 2; + case SEND_GEN: + return 2; + case SETUP_ANNOTATIONS: + return 0; + case SET_ADD: + return 2 + (oparg-1); + case SET_FUNCTION_ATTRIBUTE: + return 2; + case SET_UPDATE: + return 2 + (oparg-1); + case STORE_ATTR: + return 2; + case STORE_ATTR_INSTANCE_VALUE: + return 2; + case STORE_ATTR_SLOT: + return 2; + case STORE_ATTR_WITH_HINT: + return 2; + case STORE_DEREF: return 1; - case FOR_ITER: + case STORE_FAST: return 1; - case INSTRUMENTED_FOR_ITER: - return 0; - case _ITER_CHECK_LIST: + case STORE_FAST_LOAD_FAST: return 1; - case _ITER_JUMP_LIST: + case STORE_FAST_STORE_FAST: + return 2; + case STORE_GLOBAL: return 1; - case _GUARD_NOT_EXHAUSTED_LIST: + case STORE_NAME: return 1; - case _ITER_NEXT_LIST: + case STORE_SLICE: + return 4; + case STORE_SUBSCR: + return 3; + case STORE_SUBSCR_DICT: + return 3; + case STORE_SUBSCR_LIST_INT: + return 3; + case SWAP: + return 2 + (oparg-2); + case TO_BOOL: return 1; - case FOR_ITER_LIST: + case TO_BOOL_ALWAYS_TRUE: return 1; - case _ITER_CHECK_TUPLE: + case TO_BOOL_BOOL: return 1; - case _ITER_JUMP_TUPLE: + case TO_BOOL_INT: return 1; - case _GUARD_NOT_EXHAUSTED_TUPLE: + case TO_BOOL_LIST: return 1; - case _ITER_NEXT_TUPLE: + case TO_BOOL_NONE: return 1; - case FOR_ITER_TUPLE: + case TO_BOOL_STR: return 1; - case _ITER_CHECK_RANGE: + case UNARY_INVERT: return 1; - case _ITER_JUMP_RANGE: + case UNARY_NEGATIVE: return 1; - case _GUARD_NOT_EXHAUSTED_RANGE: + case UNARY_NOT: return 1; - case _ITER_NEXT_RANGE: + case UNPACK_EX: return 1; - case FOR_ITER_RANGE: + case UNPACK_SEQUENCE: return 1; - case FOR_ITER_GEN: + case UNPACK_SEQUENCE_LIST: return 1; - case BEFORE_ASYNC_WITH: + case UNPACK_SEQUENCE_TUPLE: return 1; - case BEFORE_WITH: + case UNPACK_SEQUENCE_TWO_TUPLE: return 1; case WITH_EXCEPT_START: return 4; - case SETUP_FINALLY: - return 0; - case SETUP_CLEANUP: - return 0; - case SETUP_WITH: - return 0; - case POP_BLOCK: - return 0; - case PUSH_EXC_INFO: - return 1; - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: - return 1; - case _GUARD_KEYS_VERSION: + case YIELD_VALUE: return 1; - case _LOAD_ATTR_METHOD_WITH_VALUES: - return 1; - case LOAD_ATTR_METHOD_WITH_VALUES: - return 1; - case _LOAD_ATTR_METHOD_NO_DICT: - return 1; - case LOAD_ATTR_METHOD_NO_DICT: - return 1; - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + default: + return -1; + } +} + +#endif + +extern int _PyOpcode_num_pushed(int opcode, int oparg); +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_num_pushed(int opcode, int oparg) { + switch(opcode) { + case BEFORE_ASYNC_WITH: + return 2; + case BEFORE_WITH: + return 2; + case BINARY_OP: return 1; - case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + case BINARY_OP_ADD_FLOAT: return 1; - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case BINARY_OP_ADD_INT: return 1; - case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + case BINARY_OP_ADD_UNICODE: return 1; - case _CHECK_ATTR_METHOD_LAZY_DICT: + case BINARY_OP_INPLACE_ADD_UNICODE: + return 0; + case BINARY_OP_MULTIPLY_FLOAT: return 1; - case _LOAD_ATTR_METHOD_LAZY_DICT: + case BINARY_OP_MULTIPLY_INT: return 1; - case LOAD_ATTR_METHOD_LAZY_DICT: + case BINARY_OP_SUBTRACT_FLOAT: return 1; - case INSTRUMENTED_CALL: - return 0; - case _SPECIALIZE_CALL: - return oparg + 2; - case _CALL: - return oparg + 2; - case CALL: - return oparg + 2; - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _CHECK_PEP_523: - return 0; - case _CHECK_FUNCTION_EXACT_ARGS: - return oparg + 2; - case _CHECK_STACK_SPACE: - return oparg + 2; - case _INIT_CALL_PY_EXACT_ARGS: - return oparg + 2; - case _PUSH_FRAME: + case BINARY_OP_SUBTRACT_INT: return 1; - case CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case CALL_PY_EXACT_ARGS: - return oparg + 2; - case CALL_PY_WITH_DEFAULTS: - return oparg + 2; - case CALL_TYPE_1: - return oparg + 2; - case CALL_STR_1: - return oparg + 2; - case CALL_TUPLE_1: - return oparg + 2; - case CALL_ALLOC_AND_ENTER_INIT: - return oparg + 2; - case EXIT_INIT_CHECK: + case BINARY_SLICE: return 1; - case CALL_BUILTIN_CLASS: - return oparg + 2; - case CALL_BUILTIN_O: - return oparg + 2; - case CALL_BUILTIN_FAST: - return oparg + 2; - case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return oparg + 2; - case CALL_LEN: - return oparg + 2; - case CALL_ISINSTANCE: - return oparg + 2; - case CALL_LIST_APPEND: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_O: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_NOARGS: - return oparg + 2; - case CALL_METHOD_DESCRIPTOR_FAST: - return oparg + 2; - case INSTRUMENTED_CALL_KW: - return 0; - case CALL_KW: - return oparg + 3; - case INSTRUMENTED_CALL_FUNCTION_EX: - return 0; - case CALL_FUNCTION_EX: - return ((oparg & 1) ? 1 : 0) + 3; - case MAKE_FUNCTION: + case BINARY_SUBSCR: return 1; - case SET_FUNCTION_ATTRIBUTE: - return 2; - case RETURN_GENERATOR: - return 0; - case BUILD_SLICE: - return ((oparg == 3) ? 1 : 0) + 2; - case CONVERT_VALUE: + case BINARY_SUBSCR_DICT: return 1; - case FORMAT_SIMPLE: + case BINARY_SUBSCR_GETITEM: return 1; - case FORMAT_WITH_SPEC: - return 2; - case COPY: - return (oparg-1) + 1; - case _SPECIALIZE_BINARY_OP: - return 2; - case _BINARY_OP: - return 2; - case BINARY_OP: - return 2; - case SWAP: - return (oparg-2) + 2; - case INSTRUMENTED_INSTRUCTION: - return 0; - case INSTRUMENTED_JUMP_FORWARD: - return 0; - case INSTRUMENTED_JUMP_BACKWARD: - return 0; - case INSTRUMENTED_POP_JUMP_IF_TRUE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_FALSE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NONE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: - return 0; - case EXTENDED_ARG: - return 0; - case CACHE: - return 0; - case RESERVED: - return 0; - case _GUARD_IS_TRUE_POP: + case BINARY_SUBSCR_LIST_INT: return 1; - case _GUARD_IS_FALSE_POP: + case BINARY_SUBSCR_STR_INT: return 1; - case _GUARD_IS_NONE_POP: + case BINARY_SUBSCR_TUPLE_INT: return 1; - case _GUARD_IS_NOT_NONE_POP: + case BUILD_CONST_KEY_MAP: return 1; - case _JUMP_TO_TOP: - return 0; - case _SET_IP: - return 0; - case _SAVE_RETURN_OFFSET: - return 0; - case _EXIT_TRACE: - return 0; - case _INSERT: - return oparg + 1; - case _CHECK_VALIDITY: - return 0; - default: - return -1; - } -} -#endif // NEED_OPCODE_METADATA - -extern int _PyOpcode_num_pushed(int opcode, int oparg, bool jump); -#ifdef NEED_OPCODE_METADATA -int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { - switch(opcode) { - case NOP: - return 0; - case RESUME: - return 0; - case RESUME_CHECK: - return 0; - case INSTRUMENTED_RESUME: - return 0; - case LOAD_CLOSURE: + case BUILD_LIST: return 1; - case LOAD_FAST_CHECK: + case BUILD_MAP: return 1; - case LOAD_FAST: + case BUILD_SET: return 1; - case LOAD_FAST_AND_CLEAR: + case BUILD_SLICE: return 1; - case LOAD_FAST_LOAD_FAST: - return 2; - case LOAD_CONST: + case BUILD_STRING: return 1; - case STORE_FAST: - return 0; - case STORE_FAST_MAYBE_NULL: - return 0; - case STORE_FAST_LOAD_FAST: + case BUILD_TUPLE: return 1; - case STORE_FAST_STORE_FAST: - return 0; - case POP_TOP: + case CACHE: return 0; - case PUSH_NULL: + case CALL: return 1; - case END_FOR: - return 0; - case INSTRUMENTED_END_FOR: - return 0; - case END_SEND: + case CALL_ALLOC_AND_ENTER_INIT: return 1; - case INSTRUMENTED_END_SEND: + case CALL_BOUND_METHOD_EXACT_ARGS: + return ((0) ? 1 : 0); + case CALL_BUILTIN_CLASS: return 1; - case UNARY_NEGATIVE: + case CALL_BUILTIN_FAST: return 1; - case UNARY_NOT: + case CALL_BUILTIN_FAST_WITH_KEYWORDS: return 1; - case _SPECIALIZE_TO_BOOL: + case CALL_BUILTIN_O: return 1; - case _TO_BOOL: + case CALL_FUNCTION_EX: return 1; - case TO_BOOL: + case CALL_INTRINSIC_1: return 1; - case TO_BOOL_BOOL: + case CALL_INTRINSIC_2: return 1; - case TO_BOOL_INT: + case CALL_ISINSTANCE: return 1; - case TO_BOOL_LIST: + case CALL_KW: return 1; - case TO_BOOL_NONE: + case CALL_LEN: return 1; - case TO_BOOL_STR: + case CALL_LIST_APPEND: return 1; - case TO_BOOL_ALWAYS_TRUE: + case CALL_METHOD_DESCRIPTOR_FAST: return 1; - case UNARY_INVERT: + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return 1; - case _GUARD_BOTH_INT: - return 2; - case _BINARY_OP_MULTIPLY_INT: + case CALL_METHOD_DESCRIPTOR_NOARGS: return 1; - case _BINARY_OP_ADD_INT: + case CALL_METHOD_DESCRIPTOR_O: return 1; - case _BINARY_OP_SUBTRACT_INT: + case CALL_PY_EXACT_ARGS: + return ((0) ? 1 : 0); + case CALL_PY_WITH_DEFAULTS: return 1; - case BINARY_OP_MULTIPLY_INT: + case CALL_STR_1: return 1; - case BINARY_OP_ADD_INT: + case CALL_TUPLE_1: return 1; - case BINARY_OP_SUBTRACT_INT: + case CALL_TYPE_1: return 1; - case _GUARD_BOTH_FLOAT: + case CHECK_EG_MATCH: return 2; - case _BINARY_OP_MULTIPLY_FLOAT: - return 1; - case _BINARY_OP_ADD_FLOAT: - return 1; - case _BINARY_OP_SUBTRACT_FLOAT: - return 1; - case BINARY_OP_MULTIPLY_FLOAT: - return 1; - case BINARY_OP_ADD_FLOAT: - return 1; - case BINARY_OP_SUBTRACT_FLOAT: - return 1; - case _GUARD_BOTH_UNICODE: + case CHECK_EXC_MATCH: return 2; - case _BINARY_OP_ADD_UNICODE: - return 1; - case BINARY_OP_ADD_UNICODE: - return 1; - case _BINARY_OP_INPLACE_ADD_UNICODE: - return 0; - case BINARY_OP_INPLACE_ADD_UNICODE: - return 0; - case _SPECIALIZE_BINARY_SUBSCR: + case CLEANUP_THROW: return 2; - case _BINARY_SUBSCR: - return 1; - case BINARY_SUBSCR: - return 1; - case BINARY_SLICE: + case COMPARE_OP: return 1; - case STORE_SLICE: - return 0; - case BINARY_SUBSCR_LIST_INT: + case COMPARE_OP_FLOAT: return 1; - case BINARY_SUBSCR_STR_INT: + case COMPARE_OP_INT: return 1; - case BINARY_SUBSCR_TUPLE_INT: + case COMPARE_OP_STR: return 1; - case BINARY_SUBSCR_DICT: + case CONTAINS_OP: return 1; - case BINARY_SUBSCR_GETITEM: + case CONVERT_VALUE: return 1; - case LIST_APPEND: - return (oparg-1) + 1; - case SET_ADD: - return (oparg-1) + 1; - case _SPECIALIZE_STORE_SUBSCR: - return 2; - case _STORE_SUBSCR: + case COPY: + return 2 + (oparg-1); + case COPY_FREE_VARS: return 0; - case STORE_SUBSCR: + case DELETE_ATTR: return 0; - case STORE_SUBSCR_LIST_INT: + case DELETE_DEREF: return 0; - case STORE_SUBSCR_DICT: + case DELETE_FAST: return 0; - case DELETE_SUBSCR: + case DELETE_GLOBAL: return 0; - case CALL_INTRINSIC_1: - return 1; - case CALL_INTRINSIC_2: - return 1; - case RAISE_VARARGS: + case DELETE_NAME: return 0; - case INTERPRETER_EXIT: + case DELETE_SUBSCR: return 0; - case _POP_FRAME: + case DICT_MERGE: + return 4 + (oparg - 1); + case DICT_UPDATE: + return 1 + (oparg - 1); + case END_ASYNC_FOR: return 0; - case RETURN_VALUE: + case END_FOR: return 0; - case INSTRUMENTED_RETURN_VALUE: + case END_SEND: + return 1; + case ENTER_EXECUTOR: return 0; - case RETURN_CONST: + case EXIT_INIT_CHECK: return 0; - case INSTRUMENTED_RETURN_CONST: + case EXTENDED_ARG: return 0; - case GET_AITER: + case FORMAT_SIMPLE: return 1; - case GET_ANEXT: - return 2; - case GET_AWAITABLE: + case FORMAT_WITH_SPEC: return 1; - case _SPECIALIZE_SEND: + case FOR_ITER: return 2; - case _SEND: + case FOR_ITER_GEN: return 2; - case SEND: + case FOR_ITER_LIST: return 2; - case SEND_GEN: + case FOR_ITER_RANGE: return 2; - case INSTRUMENTED_YIELD_VALUE: + case FOR_ITER_TUPLE: + return 2; + case GET_AITER: return 1; - case YIELD_VALUE: + case GET_ANEXT: + return 2; + case GET_AWAITABLE: return 1; - case POP_EXCEPT: - return 0; - case RERAISE: - return oparg; - case END_ASYNC_FOR: - return 0; - case CLEANUP_THROW: + case GET_ITER: + return 1; + case GET_LEN: return 2; - case LOAD_ASSERTION_ERROR: + case GET_YIELD_FROM_ITER: return 1; - case LOAD_BUILD_CLASS: + case IMPORT_FROM: + return 2; + case IMPORT_NAME: return 1; - case STORE_NAME: + case INSTRUMENTED_CALL: return 0; - case DELETE_NAME: + case INSTRUMENTED_CALL_FUNCTION_EX: return 0; - case _SPECIALIZE_UNPACK_SEQUENCE: - return 1; - case _UNPACK_SEQUENCE: - return oparg; - case UNPACK_SEQUENCE: - return oparg; - case UNPACK_SEQUENCE_TWO_TUPLE: - return oparg; - case UNPACK_SEQUENCE_TUPLE: - return oparg; - case UNPACK_SEQUENCE_LIST: - return oparg; - case UNPACK_EX: - return (oparg & 0xFF) + (oparg >> 8) + 1; - case _SPECIALIZE_STORE_ATTR: + case INSTRUMENTED_CALL_KW: + return 0; + case INSTRUMENTED_END_FOR: + return 0; + case INSTRUMENTED_END_SEND: return 1; - case _STORE_ATTR: + case INSTRUMENTED_FOR_ITER: return 0; - case STORE_ATTR: + case INSTRUMENTED_INSTRUCTION: return 0; - case DELETE_ATTR: + case INSTRUMENTED_JUMP_BACKWARD: return 0; - case STORE_GLOBAL: + case INSTRUMENTED_JUMP_FORWARD: return 0; - case DELETE_GLOBAL: + case INSTRUMENTED_LOAD_SUPER_ATTR: + return 1 + (oparg & 1); + case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; - case LOAD_LOCALS: - return 1; - case LOAD_FROM_DICT_OR_GLOBALS: - return 1; - case LOAD_NAME: - return 1; - case _SPECIALIZE_LOAD_GLOBAL: + case INSTRUMENTED_POP_JUMP_IF_NONE: return 0; - case _LOAD_GLOBAL: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_GLOBAL: - return (oparg & 1 ? 1 : 0) + 1; - case _GUARD_GLOBALS_VERSION: + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: return 0; - case _GUARD_BUILTINS_VERSION: + case INSTRUMENTED_POP_JUMP_IF_TRUE: return 0; - case _LOAD_GLOBAL_MODULE: - return ((oparg & 1) ? 1 : 0) + 1; - case _LOAD_GLOBAL_BUILTINS: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_GLOBAL_MODULE: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_GLOBAL_BUILTIN: - return (oparg & 1 ? 1 : 0) + 1; - case DELETE_FAST: + case INSTRUMENTED_RESUME: return 0; - case MAKE_CELL: + case INSTRUMENTED_RETURN_CONST: return 0; - case DELETE_DEREF: + case INSTRUMENTED_RETURN_VALUE: return 0; - case LOAD_FROM_DICT_OR_DEREF: + case INSTRUMENTED_YIELD_VALUE: return 1; - case LOAD_DEREF: + case INTERPRETER_EXIT: + return 0; + case IS_OP: return 1; - case STORE_DEREF: + case JUMP_BACKWARD: return 0; - case COPY_FREE_VARS: + case JUMP_BACKWARD_NO_INTERRUPT: return 0; - case BUILD_STRING: - return 1; - case BUILD_TUPLE: - return 1; - case BUILD_LIST: - return 1; + case JUMP_FORWARD: + return 0; + case LIST_APPEND: + return 1 + (oparg-1); case LIST_EXTEND: - return (oparg-1) + 1; - case SET_UPDATE: - return (oparg-1) + 1; - case BUILD_SET: + return 1 + (oparg-1); + case LOAD_ASSERTION_ERROR: return 1; - case BUILD_MAP: + case LOAD_ATTR: + return 1 + (oparg & 1); + case LOAD_ATTR_CLASS: + return 1 + (oparg & 1); + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: + return 1 + ((0) ? 1 : 0); + case LOAD_ATTR_INSTANCE_VALUE: + return 1 + (oparg & 1); + case LOAD_ATTR_METHOD_LAZY_DICT: + return 1 + ((1) ? 1 : 0); + case LOAD_ATTR_METHOD_NO_DICT: + return 1 + ((1) ? 1 : 0); + case LOAD_ATTR_METHOD_WITH_VALUES: + return 1 + ((1) ? 1 : 0); + case LOAD_ATTR_MODULE: + return 1 + (oparg & 1); + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1 + ((0) ? 1 : 0); + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1 + ((0) ? 1 : 0); + case LOAD_ATTR_PROPERTY: + return 1 + ((0) ? 1 : 0); + case LOAD_ATTR_SLOT: + return 1 + (oparg & 1); + case LOAD_ATTR_WITH_HINT: + return 1 + (oparg & 1); + case LOAD_BUILD_CLASS: return 1; - case SETUP_ANNOTATIONS: - return 0; - case BUILD_CONST_KEY_MAP: + case LOAD_CONST: return 1; - case DICT_UPDATE: - return (oparg - 1) + 1; - case DICT_MERGE: - return (oparg - 1) + 4; - case MAP_ADD: - return (oparg - 1) + 1; - case INSTRUMENTED_LOAD_SUPER_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case _SPECIALIZE_LOAD_SUPER_ATTR: - return 3; - case _LOAD_SUPER_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_SUPER_ATTR: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_SUPER_METHOD: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_ZERO_SUPER_METHOD: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_ZERO_SUPER_ATTR: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_SUPER_ATTR_ATTR: + case LOAD_DEREF: return 1; - case LOAD_SUPER_ATTR_METHOD: - return 2; - case _SPECIALIZE_LOAD_ATTR: + case LOAD_FAST: return 1; - case _LOAD_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_METHOD: - return (oparg & 1 ? 1 : 0) + 1; - case _GUARD_TYPE_VERSION: + case LOAD_FAST_AND_CLEAR: return 1; - case _CHECK_MANAGED_OBJECT_HAS_VALUES: + case LOAD_FAST_CHECK: return 1; - case _LOAD_ATTR_INSTANCE_VALUE: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_INSTANCE_VALUE: - return (oparg & 1 ? 1 : 0) + 1; - case _CHECK_ATTR_MODULE: + case LOAD_FAST_LOAD_FAST: + return 2; + case LOAD_FROM_DICT_OR_DEREF: return 1; - case _LOAD_ATTR_MODULE: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_MODULE: - return (oparg & 1 ? 1 : 0) + 1; - case _CHECK_ATTR_WITH_HINT: + case LOAD_FROM_DICT_OR_GLOBALS: return 1; - case _LOAD_ATTR_WITH_HINT: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_WITH_HINT: - return (oparg & 1 ? 1 : 0) + 1; - case _LOAD_ATTR_SLOT: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_SLOT: - return (oparg & 1 ? 1 : 0) + 1; - case _CHECK_ATTR_CLASS: + case LOAD_GLOBAL: + return 1 + (oparg & 1); + case LOAD_GLOBAL_BUILTIN: + return 1 + (oparg & 1); + case LOAD_GLOBAL_MODULE: + return 1 + (oparg & 1); + case LOAD_LOCALS: return 1; - case _LOAD_ATTR_CLASS: - return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_ATTR_CLASS: - return (oparg & 1 ? 1 : 0) + 1; - case LOAD_ATTR_PROPERTY: + case LOAD_NAME: return 1; - case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: + case LOAD_SUPER_ATTR: + return 1 + (oparg & 1); + case LOAD_SUPER_ATTR_ATTR: + return 1 + ((0) ? 1 : 0); + case LOAD_SUPER_ATTR_METHOD: + return 2; + case MAKE_CELL: + return 0; + case MAKE_FUNCTION: return 1; - case _GUARD_DORV_VALUES: + case MAP_ADD: + return 1 + (oparg - 1); + case MATCH_CLASS: return 1; - case _STORE_ATTR_INSTANCE_VALUE: + case MATCH_KEYS: + return 3; + case MATCH_MAPPING: + return 2; + case MATCH_SEQUENCE: + return 2; + case NOP: return 0; - case STORE_ATTR_INSTANCE_VALUE: + case POP_EXCEPT: return 0; - case STORE_ATTR_WITH_HINT: + case POP_JUMP_IF_FALSE: return 0; - case _STORE_ATTR_SLOT: + case POP_JUMP_IF_NONE: return 0; - case STORE_ATTR_SLOT: + case POP_JUMP_IF_NOT_NONE: + return 0; + case POP_JUMP_IF_TRUE: + return 0; + case POP_TOP: return 0; - case _SPECIALIZE_COMPARE_OP: + case PUSH_EXC_INFO: return 2; - case _COMPARE_OP: - return 1; - case COMPARE_OP: - return 1; - case COMPARE_OP_FLOAT: - return 1; - case COMPARE_OP_INT: - return 1; - case COMPARE_OP_STR: - return 1; - case IS_OP: - return 1; - case CONTAINS_OP: + case PUSH_NULL: return 1; - case CHECK_EG_MATCH: - return 2; - case CHECK_EXC_MATCH: + case RAISE_VARARGS: + return 0; + case RERAISE: + return oparg; + case RESERVED: + return 0; + case RESUME: + return 0; + case RESUME_CHECK: + return 0; + case RETURN_CONST: + return 0; + case RETURN_GENERATOR: + return 0; + case RETURN_VALUE: + return 0; + case SEND: return 2; - case IMPORT_NAME: - return 1; - case IMPORT_FROM: + case SEND_GEN: return 2; - case JUMP_FORWARD: + case SETUP_ANNOTATIONS: return 0; - case JUMP_BACKWARD: + case SET_ADD: + return 1 + (oparg-1); + case SET_FUNCTION_ATTRIBUTE: + return 1; + case SET_UPDATE: + return 1 + (oparg-1); + case STORE_ATTR: return 0; - case JUMP: + case STORE_ATTR_INSTANCE_VALUE: return 0; - case JUMP_NO_INTERRUPT: + case STORE_ATTR_SLOT: return 0; - case ENTER_EXECUTOR: + case STORE_ATTR_WITH_HINT: return 0; - case _POP_JUMP_IF_FALSE: + case STORE_DEREF: return 0; - case _POP_JUMP_IF_TRUE: + case STORE_FAST: return 0; - case _IS_NONE: + case STORE_FAST_LOAD_FAST: return 1; - case POP_JUMP_IF_TRUE: + case STORE_FAST_STORE_FAST: return 0; - case POP_JUMP_IF_FALSE: + case STORE_GLOBAL: return 0; - case POP_JUMP_IF_NONE: + case STORE_NAME: return 0; - case POP_JUMP_IF_NOT_NONE: + case STORE_SLICE: return 0; - case JUMP_BACKWARD_NO_INTERRUPT: + case STORE_SUBSCR: return 0; - case GET_LEN: - return 2; - case MATCH_CLASS: + case STORE_SUBSCR_DICT: + return 0; + case STORE_SUBSCR_LIST_INT: + return 0; + case SWAP: + return 2 + (oparg-2); + case TO_BOOL: return 1; - case MATCH_MAPPING: - return 2; - case MATCH_SEQUENCE: - return 2; - case MATCH_KEYS: - return 3; - case GET_ITER: - return 1; - case GET_YIELD_FROM_ITER: - return 1; - case _SPECIALIZE_FOR_ITER: - return 1; - case _FOR_ITER: - return 2; - case _FOR_ITER_TIER_TWO: - return 2; - case FOR_ITER: - return 2; - case INSTRUMENTED_FOR_ITER: - return 0; - case _ITER_CHECK_LIST: + case TO_BOOL_ALWAYS_TRUE: return 1; - case _ITER_JUMP_LIST: + case TO_BOOL_BOOL: return 1; - case _GUARD_NOT_EXHAUSTED_LIST: + case TO_BOOL_INT: return 1; - case _ITER_NEXT_LIST: - return 2; - case FOR_ITER_LIST: - return 2; - case _ITER_CHECK_TUPLE: + case TO_BOOL_LIST: return 1; - case _ITER_JUMP_TUPLE: + case TO_BOOL_NONE: return 1; - case _GUARD_NOT_EXHAUSTED_TUPLE: + case TO_BOOL_STR: return 1; - case _ITER_NEXT_TUPLE: - return 2; - case FOR_ITER_TUPLE: - return 2; - case _ITER_CHECK_RANGE: + case UNARY_INVERT: return 1; - case _ITER_JUMP_RANGE: + case UNARY_NEGATIVE: return 1; - case _GUARD_NOT_EXHAUSTED_RANGE: + case UNARY_NOT: return 1; - case _ITER_NEXT_RANGE: - return 2; - case FOR_ITER_RANGE: - return 2; - case FOR_ITER_GEN: - return 2; - case BEFORE_ASYNC_WITH: - return 2; - case BEFORE_WITH: - return 2; + case UNPACK_EX: + return 1 + (oparg >> 8) + (oparg & 0xFF); + case UNPACK_SEQUENCE: + return oparg; + case UNPACK_SEQUENCE_LIST: + return oparg; + case UNPACK_SEQUENCE_TUPLE: + return oparg; + case UNPACK_SEQUENCE_TWO_TUPLE: + return oparg; case WITH_EXCEPT_START: return 5; - case SETUP_FINALLY: - return 0; - case SETUP_CLEANUP: - return 0; - case SETUP_WITH: - return 0; - case POP_BLOCK: - return 0; - case PUSH_EXC_INFO: - return 2; - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: - return 1; - case _GUARD_KEYS_VERSION: - return 1; - case _LOAD_ATTR_METHOD_WITH_VALUES: - return 2; - case LOAD_ATTR_METHOD_WITH_VALUES: - return 2; - case _LOAD_ATTR_METHOD_NO_DICT: - return 2; - case LOAD_ATTR_METHOD_NO_DICT: - return 2; - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: - return 1; - case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: - return 1; - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: - return 1; - case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: - return 1; - case _CHECK_ATTR_METHOD_LAZY_DICT: - return 1; - case _LOAD_ATTR_METHOD_LAZY_DICT: - return 2; - case LOAD_ATTR_METHOD_LAZY_DICT: - return 2; - case INSTRUMENTED_CALL: - return 0; - case _SPECIALIZE_CALL: - return oparg + 2; - case _CALL: - return 1; - case CALL: - return 1; - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: - return oparg + 2; - case _CHECK_PEP_523: - return 0; - case _CHECK_FUNCTION_EXACT_ARGS: - return oparg + 2; - case _CHECK_STACK_SPACE: - return oparg + 2; - case _INIT_CALL_PY_EXACT_ARGS: - return 1; - case _PUSH_FRAME: - return 1; - case CALL_BOUND_METHOD_EXACT_ARGS: - return 1; - case CALL_PY_EXACT_ARGS: - return 1; - case CALL_PY_WITH_DEFAULTS: - return 1; - case CALL_TYPE_1: - return 1; - case CALL_STR_1: - return 1; - case CALL_TUPLE_1: - return 1; - case CALL_ALLOC_AND_ENTER_INIT: - return 1; - case EXIT_INIT_CHECK: - return 0; - case CALL_BUILTIN_CLASS: - return 1; - case CALL_BUILTIN_O: - return 1; - case CALL_BUILTIN_FAST: - return 1; - case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return 1; - case CALL_LEN: - return 1; - case CALL_ISINSTANCE: - return 1; - case CALL_LIST_APPEND: - return 1; - case CALL_METHOD_DESCRIPTOR_O: - return 1; - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return 1; - case CALL_METHOD_DESCRIPTOR_NOARGS: - return 1; - case CALL_METHOD_DESCRIPTOR_FAST: - return 1; - case INSTRUMENTED_CALL_KW: - return 0; - case CALL_KW: - return 1; - case INSTRUMENTED_CALL_FUNCTION_EX: - return 0; - case CALL_FUNCTION_EX: - return 1; - case MAKE_FUNCTION: - return 1; - case SET_FUNCTION_ATTRIBUTE: - return 1; - case RETURN_GENERATOR: - return 0; - case BUILD_SLICE: - return 1; - case CONVERT_VALUE: - return 1; - case FORMAT_SIMPLE: - return 1; - case FORMAT_WITH_SPEC: - return 1; - case COPY: - return (oparg-1) + 2; - case _SPECIALIZE_BINARY_OP: - return 2; - case _BINARY_OP: - return 1; - case BINARY_OP: + case YIELD_VALUE: return 1; - case SWAP: - return (oparg-2) + 2; - case INSTRUMENTED_INSTRUCTION: - return 0; - case INSTRUMENTED_JUMP_FORWARD: - return 0; - case INSTRUMENTED_JUMP_BACKWARD: - return 0; - case INSTRUMENTED_POP_JUMP_IF_TRUE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_FALSE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NONE: - return 0; - case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: - return 0; - case EXTENDED_ARG: - return 0; - case CACHE: - return 0; - case RESERVED: - return 0; - case _GUARD_IS_TRUE_POP: - return 0; - case _GUARD_IS_FALSE_POP: - return 0; - case _GUARD_IS_NONE_POP: - return 0; - case _GUARD_IS_NOT_NONE_POP: - return 0; - case _JUMP_TO_TOP: - return 0; - case _SET_IP: - return 0; - case _SAVE_RETURN_OFFSET: - return 0; - case _EXIT_TRACE: - return 0; - case _INSERT: - return oparg + 1; - case _CHECK_VALIDITY: - return 0; default: return -1; } } -#endif // NEED_OPCODE_METADATA + +#endif enum InstructionFormat { - INSTR_FMT_IB, - INSTR_FMT_IBC, - INSTR_FMT_IBC0, - INSTR_FMT_IBC00, - INSTR_FMT_IBC000, - INSTR_FMT_IBC0000000, - INSTR_FMT_IBC00000000, - INSTR_FMT_IX, - INSTR_FMT_IXC, - INSTR_FMT_IXC0, - INSTR_FMT_IXC00, - INSTR_FMT_IXC000, + INSTR_FMT_IB = 1, + INSTR_FMT_IBC = 2, + INSTR_FMT_IBC00 = 3, + INSTR_FMT_IBC000 = 4, + INSTR_FMT_IBC00000000 = 5, + INSTR_FMT_IX = 6, + INSTR_FMT_IXC = 7, + INSTR_FMT_IXC00 = 8, + INSTR_FMT_IXC000 = 9, }; #define IS_VALID_OPCODE(OP) \ - (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \ + (((OP) >= 0) && ((OP) < 268) && \ (_PyOpcode_opcode_metadata[(OP)].valid_entry)) #define HAS_ARG_FLAG (1) @@ -1442,17 +920,6 @@ enum InstructionFormat { #define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG)) #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) -struct opcode_metadata { - bool valid_entry; - enum InstructionFormat instr_format; - int flags; -}; - -struct opcode_macro_expansion { - int nuops; - struct { int16_t uop; int8_t size; int8_t offset; } uops[12]; -}; - #define OPARG_FULL 0 #define OPARG_CACHE_1 1 #define OPARG_CACHE_2 2 @@ -1460,651 +927,430 @@ struct opcode_macro_expansion { #define OPARG_TOP 5 #define OPARG_BOTTOM 6 #define OPARG_SAVE_RETURN_OFFSET 7 +#define OPARG_REPLACED 9 -#define OPCODE_METADATA_FLAGS(OP) (_PyOpcode_opcode_metadata[(OP)].flags & (HAS_ARG_FLAG | HAS_JUMP_FLAG)) -#define SAME_OPCODE_METADATA(OP1, OP2) \ - (OPCODE_METADATA_FLAGS(OP1) == OPCODE_METADATA_FLAGS(OP2)) - -#define OPCODE_METADATA_SIZE 512 -#define OPCODE_UOP_NAME_SIZE 512 -#define OPCODE_MACRO_EXPANSION_SIZE 256 +struct opcode_metadata { + uint8_t valid_entry; + int8_t instr_format; + int16_t flags; +}; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]; +extern const struct opcode_metadata _PyOpcode_opcode_metadata[268]; #ifdef NEED_OPCODE_METADATA -const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { - [NOP] = { true, INSTR_FMT_IX, 0 }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_CLOSURE] = { true, 0, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, 0, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, 0 }, - [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, - [END_FOR] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_SEND] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, - [_SPECIALIZE_TO_BOOL] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [_TO_BOOL] = { true, INSTR_FMT_IXC0, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_BOTH_INT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [_BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [_BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [_GUARD_BOTH_FLOAT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [_BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [_BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, 0 }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, +const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { + [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [_GUARD_BOTH_UNICODE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [_BINARY_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [_STORE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [_POP_FRAME] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_SEND] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [_SEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, - [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_UNPACK_SEQUENCE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_STORE_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, - [_STORE_ATTR] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_LOAD_GLOBAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_GLOBAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [_LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_LOAD_GLOBAL_BUILTINS] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, - [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [_SPECIALIZE_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_ATTR] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_LOAD_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR] = { true, INSTR_FMT_IBC0000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_TYPE_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_MODULE] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_WITH_HINT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_CLASS] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_GUARD_DORV_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_COMPARE_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [JUMP] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_NO_INTERRUPT] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, - [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_IS_NONE] = { true, INSTR_FMT_IX, 0 }, - [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, - [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_FOR_ITER_TIER_TWO] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_GUARD_NOT_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_NEXT_LIST] = { true, INSTR_FMT_IX, 0 }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [_ITER_CHECK_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_JUMP_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_GUARD_NOT_EXHAUSTED_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_NEXT_TUPLE] = { true, INSTR_FMT_IX, 0 }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [_ITER_CHECK_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_JUMP_RANGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [_GUARD_NOT_EXHAUSTED_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_ITER_NEXT_RANGE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SETUP_FINALLY] = { true, 0, 0 }, - [SETUP_CLEANUP] = { true, 0, 0 }, - [SETUP_WITH] = { true, 0, 0 }, - [POP_BLOCK] = { true, 0, 0 }, - [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_GUARD_KEYS_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_SPECIALIZE_CALL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_CALL] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_CHECK_PEP_523] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_CHECK_FUNCTION_EXACT_ARGS] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_CHECK_STACK_SPACE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [_INIT_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_PUSH_FRAME] = { true, INSTR_FMT_IX, 0 }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, 0 }, + [END_SEND] = { true, INSTR_FMT_IX, 0 }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_SPECIALIZE_BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_BINARY_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, + [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, + [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, + [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, + [NOP] = { true, INSTR_FMT_IX, 0 }, + [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_TOP] = { true, INSTR_FMT_IX, 0 }, + [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, + [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [_GUARD_IS_TRUE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_GUARD_IS_FALSE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_GUARD_IS_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_GUARD_IS_NOT_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, - [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_SAVE_RETURN_OFFSET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_EXIT_TRACE] = { true, INSTR_FMT_IX, 0 }, - [_INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_CHECK_VALIDITY] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, + [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ZERO_SUPER_ATTR] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ZERO_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [POP_BLOCK] = { true, -1, 0 }, + [SETUP_CLEANUP] = { true, -1, HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_ARG_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, }; -#endif // NEED_OPCODE_METADATA +#endif + +#define MAX_UOP_PER_EXPANSION 8 +struct opcode_macro_expansion { + int nuops; + struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; +}; +extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; -extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]; #ifdef NEED_OPCODE_METADATA -const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { - [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, - [RESUME_CHECK] = { .nuops = 1, .uops = { { RESUME_CHECK, 0, 0 } } }, - [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, - [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, - [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } }, - [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { LOAD_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, - [LOAD_CONST] = { .nuops = 1, .uops = { { LOAD_CONST, 0, 0 } } }, - [STORE_FAST] = { .nuops = 1, .uops = { { STORE_FAST, 0, 0 } } }, - [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, - [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { STORE_FAST, 6, 0 } } }, - [POP_TOP] = { .nuops = 1, .uops = { { POP_TOP, 0, 0 } } }, - [PUSH_NULL] = { .nuops = 1, .uops = { { PUSH_NULL, 0, 0 } } }, - [END_FOR] = { .nuops = 2, .uops = { { POP_TOP, 0, 0 }, { POP_TOP, 0, 0 } } }, - [END_SEND] = { .nuops = 1, .uops = { { END_SEND, 0, 0 } } }, - [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { UNARY_NEGATIVE, 0, 0 } } }, - [UNARY_NOT] = { .nuops = 1, .uops = { { UNARY_NOT, 0, 0 } } }, - [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, - [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { TO_BOOL_BOOL, 0, 0 } } }, - [TO_BOOL_INT] = { .nuops = 1, .uops = { { TO_BOOL_INT, 0, 0 } } }, - [TO_BOOL_LIST] = { .nuops = 1, .uops = { { TO_BOOL_LIST, 0, 0 } } }, - [TO_BOOL_NONE] = { .nuops = 1, .uops = { { TO_BOOL_NONE, 0, 0 } } }, - [TO_BOOL_STR] = { .nuops = 1, .uops = { { TO_BOOL_STR, 0, 0 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, - [UNARY_INVERT] = { .nuops = 1, .uops = { { UNARY_INVERT, 0, 0 } } }, - [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, +const struct opcode_macro_expansion +_PyOpcode_macro_expansion[256] = { + [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { _BEFORE_ASYNC_WITH, 0, 0 } } }, + [BEFORE_WITH] = { .nuops = 1, .uops = { { _BEFORE_WITH, 0, 0 } } }, + [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, + [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, - [BINARY_OP_SUBTRACT_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_SUBTRACT_INT, 0, 0 } } }, + [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, 0, 0 } } }, - [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, + [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } }, - [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, + [BINARY_OP_SUBTRACT_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_SUBTRACT_INT, 0, 0 } } }, + [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, 0, 0 } } }, [BINARY_SUBSCR] = { .nuops = 1, .uops = { { _BINARY_SUBSCR, 0, 0 } } }, - [BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } }, - [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } }, - [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } }, - [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_STR_INT, 0, 0 } } }, - [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, - [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, - [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, - [SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } }, - [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, 0, 0 } } }, - [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } }, - [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } }, - [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, - [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { CALL_INTRINSIC_1, 0, 0 } } }, - [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { CALL_INTRINSIC_2, 0, 0 } } }, - [RETURN_VALUE] = { .nuops = 1, .uops = { { _POP_FRAME, 0, 0 } } }, - [RETURN_CONST] = { .nuops = 2, .uops = { { LOAD_CONST, 0, 0 }, { _POP_FRAME, 0, 0 } } }, - [GET_AITER] = { .nuops = 1, .uops = { { GET_AITER, 0, 0 } } }, - [GET_ANEXT] = { .nuops = 1, .uops = { { GET_ANEXT, 0, 0 } } }, - [GET_AWAITABLE] = { .nuops = 1, .uops = { { GET_AWAITABLE, 0, 0 } } }, - [POP_EXCEPT] = { .nuops = 1, .uops = { { POP_EXCEPT, 0, 0 } } }, - [LOAD_ASSERTION_ERROR] = { .nuops = 1, .uops = { { LOAD_ASSERTION_ERROR, 0, 0 } } }, - [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { LOAD_BUILD_CLASS, 0, 0 } } }, - [STORE_NAME] = { .nuops = 1, .uops = { { STORE_NAME, 0, 0 } } }, - [DELETE_NAME] = { .nuops = 1, .uops = { { DELETE_NAME, 0, 0 } } }, - [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, 0, 0 } } }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, - [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, - [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } }, - [UNPACK_EX] = { .nuops = 1, .uops = { { UNPACK_EX, 0, 0 } } }, - [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, - [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, - [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, - [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, - [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } }, - [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, - [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } }, - [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, - [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, - [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, - [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, - [MAKE_CELL] = { .nuops = 1, .uops = { { MAKE_CELL, 0, 0 } } }, - [DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } }, - [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, - [LOAD_DEREF] = { .nuops = 1, .uops = { { LOAD_DEREF, 0, 0 } } }, - [STORE_DEREF] = { .nuops = 1, .uops = { { STORE_DEREF, 0, 0 } } }, - [COPY_FREE_VARS] = { .nuops = 1, .uops = { { COPY_FREE_VARS, 0, 0 } } }, - [BUILD_STRING] = { .nuops = 1, .uops = { { BUILD_STRING, 0, 0 } } }, - [BUILD_TUPLE] = { .nuops = 1, .uops = { { BUILD_TUPLE, 0, 0 } } }, - [BUILD_LIST] = { .nuops = 1, .uops = { { BUILD_LIST, 0, 0 } } }, - [LIST_EXTEND] = { .nuops = 1, .uops = { { LIST_EXTEND, 0, 0 } } }, - [SET_UPDATE] = { .nuops = 1, .uops = { { SET_UPDATE, 0, 0 } } }, - [BUILD_SET] = { .nuops = 1, .uops = { { BUILD_SET, 0, 0 } } }, - [BUILD_MAP] = { .nuops = 1, .uops = { { BUILD_MAP, 0, 0 } } }, - [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { SETUP_ANNOTATIONS, 0, 0 } } }, - [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { BUILD_CONST_KEY_MAP, 0, 0 } } }, - [DICT_UPDATE] = { .nuops = 1, .uops = { { DICT_UPDATE, 0, 0 } } }, - [DICT_MERGE] = { .nuops = 1, .uops = { { DICT_MERGE, 0, 0 } } }, - [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } }, - [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, - [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, + [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_DICT, 0, 0 } } }, + [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_LIST_INT, 0, 0 } } }, + [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_STR_INT, 0, 0 } } }, + [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, + [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } }, + [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } }, + [BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } }, + [BUILD_SET] = { .nuops = 1, .uops = { { _BUILD_SET, 0, 0 } } }, + [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, + [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, + [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST, 0, 0 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, + [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { _CALL_BUILTIN_O, 0, 0 } } }, + [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, + [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, + [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, + [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_STR_1] = { .nuops = 1, .uops = { { _CALL_STR_1, 0, 0 } } }, + [CALL_TUPLE_1] = { .nuops = 1, .uops = { { _CALL_TUPLE_1, 0, 0 } } }, + [CALL_TYPE_1] = { .nuops = 1, .uops = { { _CALL_TYPE_1, 0, 0 } } }, + [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, 0, 0 } } }, + [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, 0, 0 } } }, + [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, 0, 0 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { _COMPARE_OP_FLOAT, 0, 0 } } }, + [COMPARE_OP_INT] = { .nuops = 1, .uops = { { _COMPARE_OP_INT, 0, 0 } } }, + [COMPARE_OP_STR] = { .nuops = 1, .uops = { { _COMPARE_OP_STR, 0, 0 } } }, + [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, 0, 0 } } }, + [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, 0, 0 } } }, + [COPY] = { .nuops = 1, .uops = { { _COPY, 0, 0 } } }, + [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, 0, 0 } } }, + [DELETE_ATTR] = { .nuops = 1, .uops = { { _DELETE_ATTR, 0, 0 } } }, + [DELETE_DEREF] = { .nuops = 1, .uops = { { _DELETE_DEREF, 0, 0 } } }, + [DELETE_FAST] = { .nuops = 1, .uops = { { _DELETE_FAST, 0, 0 } } }, + [DELETE_GLOBAL] = { .nuops = 1, .uops = { { _DELETE_GLOBAL, 0, 0 } } }, + [DELETE_NAME] = { .nuops = 1, .uops = { { _DELETE_NAME, 0, 0 } } }, + [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, 0, 0 } } }, + [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, 0, 0 } } }, + [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, 0, 0 } } }, + [END_FOR] = { .nuops = 2, .uops = { { _POP_TOP, 0, 0 }, { _POP_TOP, 0, 0 } } }, + [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, 0, 0 } } }, + [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, 0, 0 } } }, + [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, 0, 0 } } }, + [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, 0, 0 } } }, + [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, 9, 0 } } }, + [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, 0, 0 }, { _ITER_JUMP_LIST, 9, 1 }, { _ITER_NEXT_LIST, 0, 0 } } }, + [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, 0, 0 }, { _ITER_JUMP_RANGE, 9, 1 }, { _ITER_NEXT_RANGE, 0, 0 } } }, + [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, 0, 0 }, { _ITER_JUMP_TUPLE, 9, 1 }, { _ITER_NEXT_TUPLE, 0, 0 } } }, + [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, 0, 0 } } }, + [GET_ANEXT] = { .nuops = 1, .uops = { { _GET_ANEXT, 0, 0 } } }, + [GET_AWAITABLE] = { .nuops = 1, .uops = { { _GET_AWAITABLE, 0, 0 } } }, + [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, 0, 0 } } }, + [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, 0, 0 } } }, + [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, 0, 0 } } }, + [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, 0, 0 } } }, + [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, 0, 0 } } }, + [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, 0, 0 } } }, + [LOAD_ASSERTION_ERROR] = { .nuops = 1, .uops = { { _LOAD_ASSERTION_ERROR, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, - [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, - [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, + [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, 0, 0 } } }, + [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, 0, 0 } } }, + [LOAD_DEREF] = { .nuops = 1, .uops = { { _LOAD_DEREF, 0, 0 } } }, + [LOAD_FAST] = { .nuops = 1, .uops = { { _LOAD_FAST, 0, 0 } } }, + [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { _LOAD_FAST_AND_CLEAR, 0, 0 } } }, + [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { _LOAD_FAST_CHECK, 0, 0 } } }, + [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, + [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, + [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, + [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, + [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, + [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, + [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, + [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } }, + [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, + [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, + [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, 0, 0 } } }, + [MAKE_FUNCTION] = { .nuops = 1, .uops = { { _MAKE_FUNCTION, 0, 0 } } }, + [MAP_ADD] = { .nuops = 1, .uops = { { _MAP_ADD, 0, 0 } } }, + [MATCH_CLASS] = { .nuops = 1, .uops = { { _MATCH_CLASS, 0, 0 } } }, + [MATCH_KEYS] = { .nuops = 1, .uops = { { _MATCH_KEYS, 0, 0 } } }, + [MATCH_MAPPING] = { .nuops = 1, .uops = { { _MATCH_MAPPING, 0, 0 } } }, + [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { _MATCH_SEQUENCE, 0, 0 } } }, + [NOP] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, + [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, 0, 0 } } }, + [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 9, 1 } } }, + [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 9, 1 } } }, + [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_FALSE, 9, 1 } } }, + [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, 9, 1 } } }, + [POP_TOP] = { .nuops = 1, .uops = { { _POP_TOP, 0, 0 } } }, + [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, 0, 0 } } }, + [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, 0, 0 } } }, + [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, 0, 0 } } }, + [RETURN_CONST] = { .nuops = 2, .uops = { { _LOAD_CONST, 0, 0 }, { _POP_FRAME, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 1, .uops = { { _POP_FRAME, 0, 0 } } }, + [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, 0, 0 } } }, + [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, 0, 0 } } }, + [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, + [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } }, + [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, - [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, 0, 0 } } }, - [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, - [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, - [COMPARE_OP_STR] = { .nuops = 1, .uops = { { COMPARE_OP_STR, 0, 0 } } }, - [IS_OP] = { .nuops = 1, .uops = { { IS_OP, 0, 0 } } }, - [CONTAINS_OP] = { .nuops = 1, .uops = { { CONTAINS_OP, 0, 0 } } }, - [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { CHECK_EG_MATCH, 0, 0 } } }, - [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { CHECK_EXC_MATCH, 0, 0 } } }, - [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, 0, 0 } } }, - [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 0, 0 } } }, - [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 0, 0 } } }, - [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_FALSE, 0, 0 } } }, - [GET_LEN] = { .nuops = 1, .uops = { { GET_LEN, 0, 0 } } }, - [MATCH_CLASS] = { .nuops = 1, .uops = { { MATCH_CLASS, 0, 0 } } }, - [MATCH_MAPPING] = { .nuops = 1, .uops = { { MATCH_MAPPING, 0, 0 } } }, - [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { MATCH_SEQUENCE, 0, 0 } } }, - [MATCH_KEYS] = { .nuops = 1, .uops = { { MATCH_KEYS, 0, 0 } } }, - [GET_ITER] = { .nuops = 1, .uops = { { GET_ITER, 0, 0 } } }, - [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, - [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, 0, 0 } } }, - [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, 0, 0 }, { _ITER_JUMP_LIST, 0, 0 }, { _ITER_NEXT_LIST, 0, 0 } } }, - [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, 0, 0 }, { _ITER_JUMP_TUPLE, 0, 0 }, { _ITER_NEXT_TUPLE, 0, 0 } } }, - [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, 0, 0 }, { _ITER_JUMP_RANGE, 0, 0 }, { _ITER_NEXT_RANGE, 0, 0 } } }, - [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { BEFORE_ASYNC_WITH, 0, 0 } } }, - [BEFORE_WITH] = { .nuops = 1, .uops = { { BEFORE_WITH, 0, 0 } } }, - [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, - [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, - [CALL_STR_1] = { .nuops = 1, .uops = { { CALL_STR_1, 0, 0 } } }, - [CALL_TUPLE_1] = { .nuops = 1, .uops = { { CALL_TUPLE_1, 0, 0 } } }, - [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { EXIT_INIT_CHECK, 0, 0 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { CALL_BUILTIN_CLASS, 0, 0 } } }, - [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { CALL_BUILTIN_O, 0, 0 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST, 0, 0 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_LEN] = { .nuops = 1, .uops = { { CALL_LEN, 0, 0 } } }, - [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { CALL_ISINSTANCE, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, - [MAKE_FUNCTION] = { .nuops = 1, .uops = { { MAKE_FUNCTION, 0, 0 } } }, - [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, - [BUILD_SLICE] = { .nuops = 1, .uops = { { BUILD_SLICE, 0, 0 } } }, - [CONVERT_VALUE] = { .nuops = 1, .uops = { { CONVERT_VALUE, 0, 0 } } }, - [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { FORMAT_SIMPLE, 0, 0 } } }, - [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { FORMAT_WITH_SPEC, 0, 0 } } }, - [COPY] = { .nuops = 1, .uops = { { COPY, 0, 0 } } }, - [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, - [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, -}; -#endif // NEED_OPCODE_METADATA - -extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; -#ifdef NEED_OPCODE_METADATA -const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { - [_EXIT_TRACE] = "_EXIT_TRACE", - [_SET_IP] = "_SET_IP", - [_SPECIALIZE_TO_BOOL] = "_SPECIALIZE_TO_BOOL", - [_TO_BOOL] = "_TO_BOOL", - [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", - [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", - [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", - [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", - [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", - [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", - [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", - [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", - [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", - [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", - [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", - [_SPECIALIZE_BINARY_SUBSCR] = "_SPECIALIZE_BINARY_SUBSCR", - [_BINARY_SUBSCR] = "_BINARY_SUBSCR", - [_SPECIALIZE_STORE_SUBSCR] = "_SPECIALIZE_STORE_SUBSCR", - [_STORE_SUBSCR] = "_STORE_SUBSCR", - [_POP_FRAME] = "_POP_FRAME", - [_SPECIALIZE_SEND] = "_SPECIALIZE_SEND", - [_SEND] = "_SEND", - [_SPECIALIZE_UNPACK_SEQUENCE] = "_SPECIALIZE_UNPACK_SEQUENCE", - [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", - [_SPECIALIZE_STORE_ATTR] = "_SPECIALIZE_STORE_ATTR", - [_STORE_ATTR] = "_STORE_ATTR", - [_SPECIALIZE_LOAD_GLOBAL] = "_SPECIALIZE_LOAD_GLOBAL", - [_LOAD_GLOBAL] = "_LOAD_GLOBAL", - [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", - [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", - [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", - [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", - [_SPECIALIZE_LOAD_SUPER_ATTR] = "_SPECIALIZE_LOAD_SUPER_ATTR", - [_LOAD_SUPER_ATTR] = "_LOAD_SUPER_ATTR", - [_SPECIALIZE_LOAD_ATTR] = "_SPECIALIZE_LOAD_ATTR", - [_LOAD_ATTR] = "_LOAD_ATTR", - [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", - [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", - [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", - [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", - [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", - [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", - [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", - [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", - [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", - [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", - [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", - [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", - [_SPECIALIZE_COMPARE_OP] = "_SPECIALIZE_COMPARE_OP", - [_COMPARE_OP] = "_COMPARE_OP", - [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", - [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", - [_IS_NONE] = "_IS_NONE", - [_SPECIALIZE_FOR_ITER] = "_SPECIALIZE_FOR_ITER", - [_FOR_ITER] = "_FOR_ITER", - [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", - [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", - [_ITER_JUMP_LIST] = "_ITER_JUMP_LIST", - [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", - [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", - [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", - [_ITER_JUMP_TUPLE] = "_ITER_JUMP_TUPLE", - [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", - [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", - [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", - [_ITER_JUMP_RANGE] = "_ITER_JUMP_RANGE", - [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", - [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", - [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", - [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", - [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", - [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", - [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", - [_SPECIALIZE_CALL] = "_SPECIALIZE_CALL", - [_CALL] = "_CALL", - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", - [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", - [_CHECK_PEP_523] = "_CHECK_PEP_523", - [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", - [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", - [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", - [_PUSH_FRAME] = "_PUSH_FRAME", - [_SPECIALIZE_BINARY_OP] = "_SPECIALIZE_BINARY_OP", - [_BINARY_OP] = "_BINARY_OP", - [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", - [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", - [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", - [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", - [_JUMP_TO_TOP] = "_JUMP_TO_TOP", - [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", - [_INSERT] = "_INSERT", - [_CHECK_VALIDITY] = "_CHECK_VALIDITY", -}; -#endif // NEED_OPCODE_METADATA - -extern const char *const _PyOpcode_OpName[268]; -#ifdef NEED_OPCODE_METADATA -const char *const _PyOpcode_OpName[268] = { - [CACHE] = "CACHE", - [RESERVED] = "RESERVED", - [RESUME] = "RESUME", - [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", - [BEFORE_WITH] = "BEFORE_WITH", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", - [BINARY_SLICE] = "BINARY_SLICE", - [BINARY_SUBSCR] = "BINARY_SUBSCR", - [CHECK_EG_MATCH] = "CHECK_EG_MATCH", - [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", - [CLEANUP_THROW] = "CLEANUP_THROW", - [DELETE_SUBSCR] = "DELETE_SUBSCR", - [END_ASYNC_FOR] = "END_ASYNC_FOR", - [END_FOR] = "END_FOR", - [END_SEND] = "END_SEND", - [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", - [FORMAT_SIMPLE] = "FORMAT_SIMPLE", - [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", - [GET_AITER] = "GET_AITER", - [GET_ANEXT] = "GET_ANEXT", - [GET_ITER] = "GET_ITER", - [GET_LEN] = "GET_LEN", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [INTERPRETER_EXIT] = "INTERPRETER_EXIT", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_LOCALS] = "LOAD_LOCALS", - [MAKE_FUNCTION] = "MAKE_FUNCTION", - [MATCH_KEYS] = "MATCH_KEYS", - [MATCH_MAPPING] = "MATCH_MAPPING", - [MATCH_SEQUENCE] = "MATCH_SEQUENCE", - [NOP] = "NOP", - [POP_EXCEPT] = "POP_EXCEPT", - [POP_TOP] = "POP_TOP", - [PUSH_EXC_INFO] = "PUSH_EXC_INFO", - [PUSH_NULL] = "PUSH_NULL", - [RETURN_GENERATOR] = "RETURN_GENERATOR", - [RETURN_VALUE] = "RETURN_VALUE", - [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [STORE_SLICE] = "STORE_SLICE", - [STORE_SUBSCR] = "STORE_SUBSCR", - [TO_BOOL] = "TO_BOOL", - [UNARY_INVERT] = "UNARY_INVERT", - [UNARY_NEGATIVE] = "UNARY_NEGATIVE", - [UNARY_NOT] = "UNARY_NOT", - [WITH_EXCEPT_START] = "WITH_EXCEPT_START", + [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } }, + [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } }, + [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, + [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, 5, 0 }, { _STORE_FAST, 6, 0 } } }, + [STORE_GLOBAL] = { .nuops = 1, .uops = { { _STORE_GLOBAL, 0, 0 } } }, + [STORE_NAME] = { .nuops = 1, .uops = { { _STORE_NAME, 0, 0 } } }, + [STORE_SLICE] = { .nuops = 1, .uops = { { _STORE_SLICE, 0, 0 } } }, + [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, 0, 0 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_DICT, 0, 0 } } }, + [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_LIST_INT, 0, 0 } } }, + [SWAP] = { .nuops = 1, .uops = { { _SWAP, 0, 0 } } }, + [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { _TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, + [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, 0, 0 } } }, + [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, 0, 0 } } }, + [TO_BOOL_LIST] = { .nuops = 1, .uops = { { _TO_BOOL_LIST, 0, 0 } } }, + [TO_BOOL_NONE] = { .nuops = 1, .uops = { { _TO_BOOL_NONE, 0, 0 } } }, + [TO_BOOL_STR] = { .nuops = 1, .uops = { { _TO_BOOL_STR, 0, 0 } } }, + [UNARY_INVERT] = { .nuops = 1, .uops = { { _UNARY_INVERT, 0, 0 } } }, + [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { _UNARY_NEGATIVE, 0, 0 } } }, + [UNARY_NOT] = { .nuops = 1, .uops = { { _UNARY_NOT, 0, 0 } } }, + [UNPACK_EX] = { .nuops = 1, .uops = { { _UNPACK_EX, 0, 0 } } }, + [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, 0, 0 } } }, + [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_LIST, 0, 0 } } }, + [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, + [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, 0, 0 } } }, +}; +#endif // NEED_OPCODE_METADATA + +extern const char *_PyOpcode_OpName[268]; +#ifdef NEED_OPCODE_METADATA +const char *_PyOpcode_OpName[268] = { + [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", + [BEFORE_WITH] = "BEFORE_WITH", [BINARY_OP] = "BINARY_OP", + [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", + [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", + [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", + [BINARY_SLICE] = "BINARY_SLICE", + [BINARY_SUBSCR] = "BINARY_SUBSCR", + [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", + [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", + [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", + [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", + [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_LIST] = "BUILD_LIST", [BUILD_MAP] = "BUILD_MAP", @@ -2112,12 +1358,37 @@ const char *const _PyOpcode_OpName[268] = { [BUILD_SLICE] = "BUILD_SLICE", [BUILD_STRING] = "BUILD_STRING", [BUILD_TUPLE] = "BUILD_TUPLE", + [CACHE] = "CACHE", [CALL] = "CALL", + [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", + [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_O] = "CALL_BUILTIN_O", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", + [CALL_ISINSTANCE] = "CALL_ISINSTANCE", [CALL_KW] = "CALL_KW", + [CALL_LEN] = "CALL_LEN", + [CALL_LIST_APPEND] = "CALL_LIST_APPEND", + [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", + [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", + [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [CALL_STR_1] = "CALL_STR_1", + [CALL_TUPLE_1] = "CALL_TUPLE_1", + [CALL_TYPE_1] = "CALL_TYPE_1", + [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", + [CLEANUP_THROW] = "CLEANUP_THROW", [COMPARE_OP] = "COMPARE_OP", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [COMPARE_OP_INT] = "COMPARE_OP_INT", + [COMPARE_OP_STR] = "COMPARE_OP_STR", [CONTAINS_OP] = "CONTAINS_OP", [CONVERT_VALUE] = "CONVERT_VALUE", [COPY] = "COPY", @@ -2127,21 +1398,74 @@ const char *const _PyOpcode_OpName[268] = { [DELETE_FAST] = "DELETE_FAST", [DELETE_GLOBAL] = "DELETE_GLOBAL", [DELETE_NAME] = "DELETE_NAME", + [DELETE_SUBSCR] = "DELETE_SUBSCR", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [END_ASYNC_FOR] = "END_ASYNC_FOR", + [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", [ENTER_EXECUTOR] = "ENTER_EXECUTOR", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", [EXTENDED_ARG] = "EXTENDED_ARG", + [FORMAT_SIMPLE] = "FORMAT_SIMPLE", + [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", [FOR_ITER] = "FOR_ITER", + [FOR_ITER_GEN] = "FOR_ITER_GEN", + [FOR_ITER_LIST] = "FOR_ITER_LIST", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [GET_AITER] = "GET_AITER", + [GET_ANEXT] = "GET_ANEXT", [GET_AWAITABLE] = "GET_AWAITABLE", + [GET_ITER] = "GET_ITER", + [GET_LEN] = "GET_LEN", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [IMPORT_FROM] = "IMPORT_FROM", [IMPORT_NAME] = "IMPORT_NAME", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", + [INSTRUMENTED_CALL_KW] = "INSTRUMENTED_CALL_KW", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", + [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [IS_OP] = "IS_OP", + [JUMP] = "JUMP", [JUMP_BACKWARD] = "JUMP_BACKWARD", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [JUMP_FORWARD] = "JUMP_FORWARD", + [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", [LIST_APPEND] = "LIST_APPEND", [LIST_EXTEND] = "LIST_EXTEND", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [LOAD_ATTR] = "LOAD_ATTR", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_CLOSURE] = "LOAD_CLOSURE", [LOAD_CONST] = "LOAD_CONST", [LOAD_DEREF] = "LOAD_DEREF", [LOAD_FAST] = "LOAD_FAST", @@ -2151,141 +1475,92 @@ const char *const _PyOpcode_OpName[268] = { [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", [LOAD_GLOBAL] = "LOAD_GLOBAL", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_LOCALS] = "LOAD_LOCALS", + [LOAD_METHOD] = "LOAD_METHOD", [LOAD_NAME] = "LOAD_NAME", [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", + [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", + [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", + [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", + [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", + [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", [MAKE_CELL] = "MAKE_CELL", + [MAKE_FUNCTION] = "MAKE_FUNCTION", [MAP_ADD] = "MAP_ADD", [MATCH_CLASS] = "MATCH_CLASS", + [MATCH_KEYS] = "MATCH_KEYS", + [MATCH_MAPPING] = "MATCH_MAPPING", + [MATCH_SEQUENCE] = "MATCH_SEQUENCE", + [NOP] = "NOP", + [POP_BLOCK] = "POP_BLOCK", + [POP_EXCEPT] = "POP_EXCEPT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", + [POP_TOP] = "POP_TOP", + [PUSH_EXC_INFO] = "PUSH_EXC_INFO", + [PUSH_NULL] = "PUSH_NULL", [RAISE_VARARGS] = "RAISE_VARARGS", [RERAISE] = "RERAISE", + [RESERVED] = "RESERVED", + [RESUME] = "RESUME", + [RESUME_CHECK] = "RESUME_CHECK", [RETURN_CONST] = "RETURN_CONST", + [RETURN_GENERATOR] = "RETURN_GENERATOR", + [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", + [SEND_GEN] = "SEND_GEN", + [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [SETUP_CLEANUP] = "SETUP_CLEANUP", + [SETUP_FINALLY] = "SETUP_FINALLY", + [SETUP_WITH] = "SETUP_WITH", [SET_ADD] = "SET_ADD", [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", [SET_UPDATE] = "SET_UPDATE", [STORE_ATTR] = "STORE_ATTR", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_DEREF] = "STORE_DEREF", [STORE_FAST] = "STORE_FAST", [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", + [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", [STORE_GLOBAL] = "STORE_GLOBAL", [STORE_NAME] = "STORE_NAME", - [SWAP] = "SWAP", - [UNPACK_EX] = "UNPACK_EX", - [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", - [YIELD_VALUE] = "YIELD_VALUE", - [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", - [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", - [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", - [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", - [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", - [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", - [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", - [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", - [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", - [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_BUILTIN_O] = "CALL_BUILTIN_O", - [CALL_ISINSTANCE] = "CALL_ISINSTANCE", - [CALL_LEN] = "CALL_LEN", - [CALL_LIST_APPEND] = "CALL_LIST_APPEND", - [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", - [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", - [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_STR_1] = "CALL_STR_1", - [CALL_TUPLE_1] = "CALL_TUPLE_1", - [CALL_TYPE_1] = "CALL_TYPE_1", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", - [COMPARE_OP_INT] = "COMPARE_OP_INT", - [COMPARE_OP_STR] = "COMPARE_OP_STR", - [FOR_ITER_GEN] = "FOR_ITER_GEN", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", - [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [RESUME_CHECK] = "RESUME_CHECK", - [SEND_GEN] = "SEND_GEN", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_SLICE] = "STORE_SLICE", + [STORE_SUBSCR] = "STORE_SUBSCR", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [SWAP] = "SWAP", + [TO_BOOL] = "TO_BOOL", [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", [TO_BOOL_BOOL] = "TO_BOOL_BOOL", [TO_BOOL_INT] = "TO_BOOL_INT", [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [UNARY_INVERT] = "UNARY_INVERT", + [UNARY_NEGATIVE] = "UNARY_NEGATIVE", + [UNARY_NOT] = "UNARY_NOT", + [UNPACK_EX] = "UNPACK_EX", + [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", - [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", - [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", - [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", - [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", - [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", - [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", - [INSTRUMENTED_CALL_KW] = "INSTRUMENTED_CALL_KW", - [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", - [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", - [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", - [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", - [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", - [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", - [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", - [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [JUMP] = "JUMP", - [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", - [LOAD_CLOSURE] = "LOAD_CLOSURE", - [LOAD_METHOD] = "LOAD_METHOD", - [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", - [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", - [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", - [POP_BLOCK] = "POP_BLOCK", - [SETUP_CLEANUP] = "SETUP_CLEANUP", - [SETUP_FINALLY] = "SETUP_FINALLY", - [SETUP_WITH] = "SETUP_WITH", - [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", + [WITH_EXCEPT_START] = "WITH_EXCEPT_START", + [YIELD_VALUE] = "YIELD_VALUE", }; -#endif // NEED_OPCODE_METADATA +#endif extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { + [JUMP_BACKWARD] = 1, [TO_BOOL] = 3, - [BINARY_OP_INPLACE_ADD_UNICODE] = 1, [BINARY_SUBSCR] = 1, [STORE_SUBSCR] = 1, [SEND] = 1, @@ -2295,7 +1570,6 @@ const uint8_t _PyOpcode_Caches[256] = { [LOAD_SUPER_ATTR] = 1, [LOAD_ATTR] = 9, [COMPARE_OP] = 1, - [JUMP_BACKWARD] = 1, [POP_JUMP_IF_TRUE] = 1, [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, @@ -2304,7 +1578,7 @@ const uint8_t _PyOpcode_Caches[256] = { [CALL] = 3, [BINARY_OP] = 1, }; -#endif // NEED_OPCODE_METADATA +#endif extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA @@ -2518,6 +1792,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [WITH_EXCEPT_START] = WITH_EXCEPT_START, [YIELD_VALUE] = YIELD_VALUE, }; + #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ @@ -2570,4 +1845,40 @@ const uint8_t _PyOpcode_Deopt[256] = { case 235: \ case 255: \ ; +struct pseudo_targets { + uint8_t targets[3]; +}; +extern const struct pseudo_targets _PyOpcode_PseudoTargets[12]; +#ifdef NEED_OPCODE_METADATA +const struct pseudo_targets _PyOpcode_PseudoTargets[12] = { + [LOAD_CLOSURE-256] = { { LOAD_FAST, 0, 0 } }, + [STORE_FAST_MAYBE_NULL-256] = { { STORE_FAST, 0, 0 } }, + [LOAD_SUPER_METHOD-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, + [LOAD_ZERO_SUPER_METHOD-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, + [LOAD_ZERO_SUPER_ATTR-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, + [LOAD_METHOD-256] = { { LOAD_ATTR, 0, 0 } }, + [JUMP-256] = { { JUMP_FORWARD, JUMP_BACKWARD, 0 } }, + [JUMP_NO_INTERRUPT-256] = { { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0 } }, + [SETUP_FINALLY-256] = { { NOP, 0, 0 } }, + [SETUP_CLEANUP-256] = { { NOP, 0, 0 } }, + [SETUP_WITH-256] = { { NOP, 0, 0 } }, + [POP_BLOCK-256] = { { NOP, 0, 0 } }, +}; +#endif // NEED_OPCODE_METADATA +static inline bool +is_pseudo_target(int pseudo, int target) { + if (pseudo < 256 || pseudo >= 268) { + return false; + } + for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) { + if (_PyOpcode_PseudoTargets[pseudo-256].targets[i] == target) return true; + } + return false; +} + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CORE_OPCODE_METADATA_H */ diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index c3b72d90de3a69..0ce08900e96f0b 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -5,8 +5,20 @@ # error "this header requires Py_BUILD_CORE define" #endif -// Similar to _Py_HashPointer(), but don't replace -1 with -2 -extern Py_hash_t _Py_HashPointerRaw(const void*); +// Similar to Py_HashPointer(), but don't replace -1 with -2. +static inline Py_hash_t +_Py_HashPointerRaw(const void *ptr) +{ + uintptr_t x = (uintptr_t)ptr; + Py_BUILD_ASSERT(sizeof(x) == sizeof(ptr)); + + // Bottom 3 or 4 bits are likely to be 0; rotate x by 4 to the right + // to avoid excessive hash collisions for dicts and sets. + x = (x >> 4) | (x << (8 * sizeof(uintptr_t) - 4)); + + Py_BUILD_ASSERT(sizeof(x) == sizeof(Py_hash_t)); + return (Py_hash_t)x; +} // Export for '_datetime' shared extension PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 61e0150e89009c..c675098685764c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -40,7 +40,6 @@ extern void _PySys_FiniTypes(PyInterpreterState *interp); extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); -extern PyStatus _PyTime_Init(void); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); extern int _Py_Deepfreeze_Init(void); @@ -64,7 +63,7 @@ extern void _PyArg_Fini(void); extern void _Py_FinalizeAllocatedBlocks(_PyRuntimeState *); extern PyStatus _PyGILState_Init(PyInterpreterState *interp); -extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate); +extern void _PyGILState_SetTstate(PyThreadState *tstate); extern void _PyGILState_Fini(PyInterpreterState *interp); extern void _PyGC_DumpShutdownStats(PyInterpreterState *interp); diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 6b5113714dbeb2..8631ca34a5e616 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_PYMEM_H #define Py_INTERNAL_PYMEM_H + +#include "pycore_lock.h" // PyMutex + #ifdef __cplusplus extern "C" { #endif @@ -30,7 +33,7 @@ typedef struct { } debug_alloc_api_t; struct _pymem_allocators { - PyThread_type_lock mutex; + PyMutex mutex; struct { PyMemAllocatorEx raw; PyMemAllocatorEx mem; diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 11fbe16fa6f1d5..360fb9218a9cda 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -18,7 +18,19 @@ extern void * _PyMem_RawRealloc(void *, void *, size_t); extern void _PyMem_RawFree(void *, void *); #define PYRAW_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree} -#if defined(WITH_PYMALLOC) +#ifdef Py_GIL_DISABLED +// Py_GIL_DISABLED requires mimalloc +extern void* _PyObject_MiMalloc(void *, size_t); +extern void* _PyObject_MiCalloc(void *, size_t, size_t); +extern void _PyObject_MiFree(void *, void *); +extern void* _PyObject_MiRealloc(void *, void *, size_t); +# define PYOBJ_ALLOC {NULL, _PyObject_MiMalloc, _PyObject_MiCalloc, _PyObject_MiRealloc, _PyObject_MiFree} +extern void* _PyMem_MiMalloc(void *, size_t); +extern void* _PyMem_MiCalloc(void *, size_t, size_t); +extern void _PyMem_MiFree(void *, void *); +extern void* _PyMem_MiRealloc(void *, void *, size_t); +# define PYMEM_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree} +#elif defined(WITH_PYMALLOC) extern void* _PyObject_Malloc(void *, size_t); extern void* _PyObject_Calloc(void *, size_t, size_t); extern void _PyObject_Free(void *, void *); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 7fa952e371d7b4..37b45faf8a74a0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -187,6 +187,7 @@ extern PyThreadState * _PyThreadState_New( int whence); extern void _PyThreadState_Bind(PyThreadState *tstate); extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); +extern void _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyThreadState_GetDict(PyThreadState *tstate); @@ -220,9 +221,9 @@ PyAPI_FUNC(int) _PyState_AddModule( extern int _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_LOCK(runtime) \ - PyThread_acquire_lock((runtime)->interpreters.mutex, WAIT_LOCK) + PyMutex_LockFlags(&(runtime)->interpreters.mutex, _Py_LOCK_DONT_DETACH) #define HEAD_UNLOCK(runtime) \ - PyThread_release_lock((runtime)->interpreters.mutex) + PyMutex_Unlock(&(runtime)->interpreters.mutex) // Get the configuration of the current interpreter. // The caller must hold the GIL. diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index e6efe8b646e86f..e3348296ea61b7 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -21,7 +21,6 @@ extern "C" { #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_signal.h" // struct _signals_runtime_state -#include "pycore_time.h" // struct _time_runtime_state #include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state @@ -174,7 +173,7 @@ typedef struct pyruntimestate { unsigned long _finalizing_id; struct pyinterpreters { - PyThread_type_lock mutex; + PyMutex mutex; /* The linked list of interpreters, newest first. */ PyInterpreterState *head; /* The runtime's initial interpreter, which has a special role @@ -205,7 +204,6 @@ typedef struct pyruntimestate { struct _pymem_allocators allocators; struct _obmalloc_global_state obmalloc; struct pyhash_runtime_state pyhash_state; - struct _time_runtime_state time; struct _pythread_runtime_state threads; struct _signals_runtime_state signals; @@ -236,7 +234,7 @@ typedef struct pyruntimestate { Py_OpenCodeHookFunction open_code_hook; void *open_code_userdata; struct { - PyThread_type_lock mutex; + PyMutex mutex; _Py_AuditHookEntry *head; } audit_hooks; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index fa5d8114abf0d7..d324a94278839c 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -186,7 +186,12 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ }, \ - ._initial_thread = _PyThreadState_INIT, \ + ._initial_thread = _PyThreadStateImpl_INIT, \ + } + +#define _PyThreadStateImpl_INIT \ + { \ + .base = _PyThreadState_INIT, \ } #define _PyThreadState_INIT \ diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 46713f91d190ff..dabbd7b41556cd 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -52,16 +52,6 @@ extern "C" { #endif -struct _time_runtime_state { -#ifdef HAVE_TIMES - int ticks_per_second_initialized; - long ticks_per_second; -#else - int _not_used; -#endif -}; - - #ifdef __clang__ struct timeval; #endif @@ -263,13 +253,6 @@ PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); // Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. extern _PyTime_t _PyTime_Add(_PyTime_t t1, _PyTime_t t2); -// Compute ticks * mul / div. -// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -// The caller must ensure that ((div - 1) * mul) cannot overflow. -extern _PyTime_t _PyTime_MulDiv(_PyTime_t ticks, - _PyTime_t mul, - _PyTime_t div); - // Structure used by time.get_clock_info() typedef struct { const char *implementation; @@ -365,6 +348,32 @@ PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); +// --- _PyTimeFraction ------------------------------------------------------- + +typedef struct { + _PyTime_t numer; + _PyTime_t denom; +} _PyTimeFraction; + +// Set a fraction. +// Return 0 on success. +// Return -1 if the fraction is invalid. +extern int _PyTimeFraction_Set( + _PyTimeFraction *frac, + _PyTime_t numer, + _PyTime_t denom); + +// Compute ticks * frac.numer / frac.denom. +// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +extern _PyTime_t _PyTimeFraction_Mul( + _PyTime_t ticks, + const _PyTimeFraction *frac); + +// Compute a clock resolution: frac.numer / frac.denom / 1e9. +extern double _PyTimeFraction_Resolution( + const _PyTimeFraction *frac); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h new file mode 100644 index 00000000000000..856ddd5e7e5ff0 --- /dev/null +++ b/Include/internal/pycore_tstate.h @@ -0,0 +1,31 @@ +#ifndef Py_INTERNAL_TSTATE_H +#define Py_INTERNAL_TSTATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_mimalloc.h" // struct _mimalloc_thread_state + + +// Every PyThreadState is actually allocated as a _PyThreadStateImpl. The +// PyThreadState fields are exposed as part of the C API, although most fields +// are intended to be private. The _PyThreadStateImpl fields not exposed. +typedef struct _PyThreadStateImpl { + // semi-public fields are in PyThreadState. + PyThreadState base; + +#ifdef Py_GIL_DISABLED + struct _mimalloc_thread_state mimalloc; +#endif + +} _PyThreadStateImpl; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_TSTATE_H */ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index bbf8544b09f0fb..c03c3d766bef61 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -133,7 +133,9 @@ _PyType_IsReady(PyTypeObject *type) extern PyObject* _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); -extern PyObject* _Py_type_getattro(PyTypeObject *type, PyObject *name); +extern PyObject* _Py_type_getattro(PyObject *type, PyObject *name); + +extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op); extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); diff --git a/Include/internal/pycore_ucnhash.h b/Include/internal/pycore_ucnhash.h index 187dd68e7347ff..1561dfbb3150d3 100644 --- a/Include/internal/pycore_ucnhash.h +++ b/Include/internal/pycore_ucnhash.h @@ -28,6 +28,8 @@ typedef struct { } _PyUnicode_Name_CAPI; +extern _PyUnicode_Name_CAPI* _PyUnicode_GetNameCAPI(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index a0d00af92e0f5d..7ee540154b23d8 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex #include "pycore_fileutils.h" // _Py_error_handler #include "pycore_identifier.h" // _Py_Identifier #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI @@ -277,7 +278,7 @@ extern PyTypeObject _PyUnicodeASCIIIter_Type; /* --- Other API ---------------------------------------------------------- */ struct _Py_unicode_runtime_ids { - PyThread_type_lock lock; + PyMutex mutex; // next_index value must be preserved when Py_Initialize()/Py_Finalize() // is called multiple times: see _PyUnicode_FromId() implementation. Py_ssize_t next_index; diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h new file mode 100644 index 00000000000000..4a9a00ba352d33 --- /dev/null +++ b/Include/internal/pycore_uop_ids.h @@ -0,0 +1,239 @@ +// This file is generated by Tools/cases_generator/uop_id_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + +#ifndef Py_CORE_UOP_IDS_H +#define Py_CORE_UOP_IDS_H +#ifdef __cplusplus +extern "C" { +#endif + +#define _EXIT_TRACE 300 +#define _SET_IP 301 +#define _NOP NOP +#define _RESUME_CHECK RESUME_CHECK +#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME +#define _LOAD_FAST_CHECK LOAD_FAST_CHECK +#define _LOAD_FAST LOAD_FAST +#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR +#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST +#define _LOAD_CONST LOAD_CONST +#define _STORE_FAST STORE_FAST +#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST +#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST +#define _POP_TOP POP_TOP +#define _PUSH_NULL PUSH_NULL +#define _END_SEND END_SEND +#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _UNARY_NOT UNARY_NOT +#define _TO_BOOL 302 +#define _TO_BOOL_BOOL TO_BOOL_BOOL +#define _TO_BOOL_INT TO_BOOL_INT +#define _TO_BOOL_LIST TO_BOOL_LIST +#define _TO_BOOL_NONE TO_BOOL_NONE +#define _TO_BOOL_STR TO_BOOL_STR +#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE +#define _UNARY_INVERT UNARY_INVERT +#define _GUARD_BOTH_INT 303 +#define _BINARY_OP_MULTIPLY_INT 304 +#define _BINARY_OP_ADD_INT 305 +#define _BINARY_OP_SUBTRACT_INT 306 +#define _GUARD_BOTH_FLOAT 307 +#define _BINARY_OP_MULTIPLY_FLOAT 308 +#define _BINARY_OP_ADD_FLOAT 309 +#define _BINARY_OP_SUBTRACT_FLOAT 310 +#define _GUARD_BOTH_UNICODE 311 +#define _BINARY_OP_ADD_UNICODE 312 +#define _BINARY_SUBSCR 313 +#define _BINARY_SLICE BINARY_SLICE +#define _STORE_SLICE STORE_SLICE +#define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT +#define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT +#define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT +#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT +#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM +#define _LIST_APPEND LIST_APPEND +#define _SET_ADD SET_ADD +#define _STORE_SUBSCR 314 +#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT +#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT +#define _DELETE_SUBSCR DELETE_SUBSCR +#define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 +#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 +#define _POP_FRAME 315 +#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE +#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST +#define _GET_AITER GET_AITER +#define _GET_ANEXT GET_ANEXT +#define _GET_AWAITABLE GET_AWAITABLE +#define _SEND 316 +#define _SEND_GEN SEND_GEN +#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE +#define _POP_EXCEPT POP_EXCEPT +#define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR +#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS +#define _STORE_NAME STORE_NAME +#define _DELETE_NAME DELETE_NAME +#define _UNPACK_SEQUENCE 317 +#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE +#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE +#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST +#define _UNPACK_EX UNPACK_EX +#define _STORE_ATTR 318 +#define _DELETE_ATTR DELETE_ATTR +#define _STORE_GLOBAL STORE_GLOBAL +#define _DELETE_GLOBAL DELETE_GLOBAL +#define _LOAD_LOCALS LOAD_LOCALS +#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS +#define _LOAD_NAME LOAD_NAME +#define _LOAD_GLOBAL 319 +#define _GUARD_GLOBALS_VERSION 320 +#define _GUARD_BUILTINS_VERSION 321 +#define _LOAD_GLOBAL_MODULE 322 +#define _LOAD_GLOBAL_BUILTINS 323 +#define _DELETE_FAST DELETE_FAST +#define _MAKE_CELL MAKE_CELL +#define _DELETE_DEREF DELETE_DEREF +#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF +#define _LOAD_DEREF LOAD_DEREF +#define _STORE_DEREF STORE_DEREF +#define _COPY_FREE_VARS COPY_FREE_VARS +#define _BUILD_STRING BUILD_STRING +#define _BUILD_TUPLE BUILD_TUPLE +#define _BUILD_LIST BUILD_LIST +#define _LIST_EXTEND LIST_EXTEND +#define _SET_UPDATE SET_UPDATE +#define _BUILD_SET BUILD_SET +#define _BUILD_MAP BUILD_MAP +#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS +#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP +#define _DICT_UPDATE DICT_UPDATE +#define _DICT_MERGE DICT_MERGE +#define _MAP_ADD MAP_ADD +#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR +#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD +#define _LOAD_ATTR 324 +#define _GUARD_TYPE_VERSION 325 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 326 +#define _LOAD_ATTR_INSTANCE_VALUE 327 +#define _CHECK_ATTR_MODULE 328 +#define _LOAD_ATTR_MODULE 329 +#define _CHECK_ATTR_WITH_HINT 330 +#define _LOAD_ATTR_WITH_HINT 331 +#define _LOAD_ATTR_SLOT 332 +#define _CHECK_ATTR_CLASS 333 +#define _LOAD_ATTR_CLASS 334 +#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN +#define _GUARD_DORV_VALUES 335 +#define _STORE_ATTR_INSTANCE_VALUE 336 +#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT +#define _STORE_ATTR_SLOT 337 +#define _COMPARE_OP 338 +#define _COMPARE_OP_FLOAT COMPARE_OP_FLOAT +#define _COMPARE_OP_INT COMPARE_OP_INT +#define _COMPARE_OP_STR COMPARE_OP_STR +#define _IS_OP IS_OP +#define _CONTAINS_OP CONTAINS_OP +#define _CHECK_EG_MATCH CHECK_EG_MATCH +#define _CHECK_EXC_MATCH CHECK_EXC_MATCH +#define _JUMP_BACKWARD JUMP_BACKWARD +#define _POP_JUMP_IF_FALSE 339 +#define _POP_JUMP_IF_TRUE 340 +#define _IS_NONE 341 +#define _GET_LEN GET_LEN +#define _MATCH_CLASS MATCH_CLASS +#define _MATCH_MAPPING MATCH_MAPPING +#define _MATCH_SEQUENCE MATCH_SEQUENCE +#define _MATCH_KEYS MATCH_KEYS +#define _GET_ITER GET_ITER +#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER +#define _FOR_ITER 342 +#define _FOR_ITER_TIER_TWO 343 +#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER +#define _ITER_CHECK_LIST 344 +#define _ITER_JUMP_LIST 345 +#define _GUARD_NOT_EXHAUSTED_LIST 346 +#define _ITER_NEXT_LIST 347 +#define _ITER_CHECK_TUPLE 348 +#define _ITER_JUMP_TUPLE 349 +#define _GUARD_NOT_EXHAUSTED_TUPLE 350 +#define _ITER_NEXT_TUPLE 351 +#define _ITER_CHECK_RANGE 352 +#define _ITER_JUMP_RANGE 353 +#define _GUARD_NOT_EXHAUSTED_RANGE 354 +#define _ITER_NEXT_RANGE 355 +#define _FOR_ITER_GEN FOR_ITER_GEN +#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH +#define _BEFORE_WITH BEFORE_WITH +#define _WITH_EXCEPT_START WITH_EXCEPT_START +#define _PUSH_EXC_INFO PUSH_EXC_INFO +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 356 +#define _GUARD_KEYS_VERSION 357 +#define _LOAD_ATTR_METHOD_WITH_VALUES 358 +#define _LOAD_ATTR_METHOD_NO_DICT 359 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 360 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 361 +#define _CHECK_ATTR_METHOD_LAZY_DICT 362 +#define _LOAD_ATTR_METHOD_LAZY_DICT 363 +#define _INSTRUMENTED_CALL INSTRUMENTED_CALL +#define _CALL 364 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 365 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 366 +#define _CHECK_PEP_523 367 +#define _CHECK_FUNCTION_EXACT_ARGS 368 +#define _CHECK_STACK_SPACE 369 +#define _INIT_CALL_PY_EXACT_ARGS 370 +#define _PUSH_FRAME 371 +#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS +#define _CALL_TYPE_1 CALL_TYPE_1 +#define _CALL_STR_1 CALL_STR_1 +#define _CALL_TUPLE_1 CALL_TUPLE_1 +#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT +#define _EXIT_INIT_CHECK EXIT_INIT_CHECK +#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS +#define _CALL_BUILTIN_O CALL_BUILTIN_O +#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS +#define _CALL_LEN CALL_LEN +#define _CALL_ISINSTANCE CALL_ISINSTANCE +#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS +#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST +#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW +#define _CALL_KW CALL_KW +#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX +#define _CALL_FUNCTION_EX CALL_FUNCTION_EX +#define _MAKE_FUNCTION MAKE_FUNCTION +#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE +#define _BUILD_SLICE BUILD_SLICE +#define _CONVERT_VALUE CONVERT_VALUE +#define _FORMAT_SIMPLE FORMAT_SIMPLE +#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC +#define _COPY COPY +#define _BINARY_OP 372 +#define _SWAP SWAP +#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION +#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD +#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD +#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE +#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE +#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE +#define _GUARD_IS_TRUE_POP 373 +#define _GUARD_IS_FALSE_POP 374 +#define _GUARD_IS_NONE_POP 375 +#define _GUARD_IS_NOT_NONE_POP 376 +#define _JUMP_TO_TOP 377 +#define _SAVE_RETURN_OFFSET 378 +#define _INSERT 379 +#define _CHECK_VALIDITY 380 +#define MAX_UOP_ID 380 + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CORE_UOP_IDS_H */ diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h new file mode 100644 index 00000000000000..300bd3baa7b377 --- /dev/null +++ b/Include/internal/pycore_uop_metadata.h @@ -0,0 +1,403 @@ +// This file is generated by Tools/cases_generator/uop_metadata_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + +#ifndef Py_CORE_UOP_METADATA_H +#define Py_CORE_UOP_METADATA_H +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "pycore_uop_ids.h" +extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; + +#ifdef NEED_OPCODE_METADATA +const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { + [_NOP] = 0, + [_RESUME_CHECK] = HAS_DEOPT_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG, + [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_POP_TOP] = 0, + [_PUSH_NULL] = 0, + [_END_SEND] = 0, + [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_UNARY_NOT] = 0, + [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG, + [_TO_BOOL_INT] = HAS_DEOPT_FLAG, + [_TO_BOOL_LIST] = HAS_DEOPT_FLAG, + [_TO_BOOL_NONE] = HAS_DEOPT_FLAG, + [_TO_BOOL_STR] = HAS_DEOPT_FLAG, + [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG, + [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG, + [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = 0, + [_BINARY_OP_ADD_FLOAT] = 0, + [_BINARY_OP_SUBTRACT_FLOAT] = 0, + [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_POP_FRAME] = HAS_ESCAPES_FLAG, + [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_POP_EXCEPT] = HAS_ESCAPES_FLAG, + [_LOAD_ASSERTION_ERROR] = 0, + [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, + [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, + [_COPY_FREE_VARS] = HAS_ARG_FLAG, + [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG, + [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG, + [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, + [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, + [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_STR] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_IS_OP] = HAS_ARG_FLAG, + [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IS_NONE] = 0, + [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MATCH_MAPPING] = 0, + [_MATCH_SEQUENCE] = 0, + [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG, + [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG, + [_ITER_NEXT_LIST] = 0, + [_ITER_CHECK_TUPLE] = HAS_DEOPT_FLAG, + [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_DEOPT_FLAG, + [_ITER_NEXT_TUPLE] = 0, + [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG, + [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG, + [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PUSH_EXC_INFO] = 0, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, + [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, + [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, + [_CHECK_PEP_523] = HAS_DEOPT_FLAG, + [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_PUSH_FRAME] = 0, + [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_COPY] = HAS_ARG_FLAG, + [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_SWAP] = HAS_ARG_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG, + [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, + [_SET_IP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, + [_EXIT_TRACE] = HAS_DEOPT_FLAG, + [_INSERT] = HAS_ARG_FLAG, + [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, +}; + +const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { + [_BEFORE_ASYNC_WITH] = "_BEFORE_ASYNC_WITH", + [_BEFORE_WITH] = "_BEFORE_WITH", + [_BINARY_OP] = "_BINARY_OP", + [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", + [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", + [_BINARY_SLICE] = "_BINARY_SLICE", + [_BINARY_SUBSCR] = "_BINARY_SUBSCR", + [_BINARY_SUBSCR_DICT] = "_BINARY_SUBSCR_DICT", + [_BINARY_SUBSCR_LIST_INT] = "_BINARY_SUBSCR_LIST_INT", + [_BINARY_SUBSCR_STR_INT] = "_BINARY_SUBSCR_STR_INT", + [_BINARY_SUBSCR_TUPLE_INT] = "_BINARY_SUBSCR_TUPLE_INT", + [_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP", + [_BUILD_LIST] = "_BUILD_LIST", + [_BUILD_MAP] = "_BUILD_MAP", + [_BUILD_SET] = "_BUILD_SET", + [_BUILD_SLICE] = "_BUILD_SLICE", + [_BUILD_STRING] = "_BUILD_STRING", + [_BUILD_TUPLE] = "_BUILD_TUPLE", + [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", + [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", + [_CALL_BUILTIN_O] = "_CALL_BUILTIN_O", + [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", + [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", + [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_LEN] = "_CALL_LEN", + [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", + [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_STR_1] = "_CALL_STR_1", + [_CALL_TUPLE_1] = "_CALL_TUPLE_1", + [_CALL_TYPE_1] = "_CALL_TYPE_1", + [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", + [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", + [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", + [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", + [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_COMPARE_OP] = "_COMPARE_OP", + [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", + [_COMPARE_OP_INT] = "_COMPARE_OP_INT", + [_COMPARE_OP_STR] = "_COMPARE_OP_STR", + [_CONTAINS_OP] = "_CONTAINS_OP", + [_CONVERT_VALUE] = "_CONVERT_VALUE", + [_COPY] = "_COPY", + [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_DELETE_ATTR] = "_DELETE_ATTR", + [_DELETE_DEREF] = "_DELETE_DEREF", + [_DELETE_FAST] = "_DELETE_FAST", + [_DELETE_GLOBAL] = "_DELETE_GLOBAL", + [_DELETE_NAME] = "_DELETE_NAME", + [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DICT_MERGE] = "_DICT_MERGE", + [_DICT_UPDATE] = "_DICT_UPDATE", + [_END_SEND] = "_END_SEND", + [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", + [_EXIT_TRACE] = "_EXIT_TRACE", + [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", + [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", + [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", + [_GET_AITER] = "_GET_AITER", + [_GET_ANEXT] = "_GET_ANEXT", + [_GET_AWAITABLE] = "_GET_AWAITABLE", + [_GET_ITER] = "_GET_ITER", + [_GET_LEN] = "_GET_LEN", + [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", + [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", + [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", + [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", + [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", + [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", + [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", + [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", + [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INSERT] = "_INSERT", + [_IS_NONE] = "_IS_NONE", + [_IS_OP] = "_IS_OP", + [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", + [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_JUMP_TO_TOP] = "_JUMP_TO_TOP", + [_LIST_APPEND] = "_LIST_APPEND", + [_LIST_EXTEND] = "_LIST_EXTEND", + [_LOAD_ASSERTION_ERROR] = "_LOAD_ASSERTION_ERROR", + [_LOAD_ATTR] = "_LOAD_ATTR", + [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", + [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", + [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", + [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", + [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_DEREF] = "_LOAD_DEREF", + [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", + [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", + [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", + [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", + [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", + [_LOAD_GLOBAL] = "_LOAD_GLOBAL", + [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_NAME] = "_LOAD_NAME", + [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", + [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_MAKE_CELL] = "_MAKE_CELL", + [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAP_ADD] = "_MAP_ADD", + [_MATCH_CLASS] = "_MATCH_CLASS", + [_MATCH_KEYS] = "_MATCH_KEYS", + [_MATCH_MAPPING] = "_MATCH_MAPPING", + [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_NOP] = "_NOP", + [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_FRAME] = "_POP_FRAME", + [_POP_TOP] = "_POP_TOP", + [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", + [_PUSH_FRAME] = "_PUSH_FRAME", + [_PUSH_NULL] = "_PUSH_NULL", + [_RESUME_CHECK] = "_RESUME_CHECK", + [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", + [_SET_ADD] = "_SET_ADD", + [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", + [_SET_IP] = "_SET_IP", + [_SET_UPDATE] = "_SET_UPDATE", + [_STORE_ATTR] = "_STORE_ATTR", + [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_DEREF] = "_STORE_DEREF", + [_STORE_FAST] = "_STORE_FAST", + [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", + [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", + [_STORE_GLOBAL] = "_STORE_GLOBAL", + [_STORE_NAME] = "_STORE_NAME", + [_STORE_SLICE] = "_STORE_SLICE", + [_STORE_SUBSCR] = "_STORE_SUBSCR", + [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", + [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", + [_SWAP] = "_SWAP", + [_TO_BOOL] = "_TO_BOOL", + [_TO_BOOL_ALWAYS_TRUE] = "_TO_BOOL_ALWAYS_TRUE", + [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", + [_TO_BOOL_INT] = "_TO_BOOL_INT", + [_TO_BOOL_LIST] = "_TO_BOOL_LIST", + [_TO_BOOL_NONE] = "_TO_BOOL_NONE", + [_TO_BOOL_STR] = "_TO_BOOL_STR", + [_UNARY_INVERT] = "_UNARY_INVERT", + [_UNARY_NEGATIVE] = "_UNARY_NEGATIVE", + [_UNARY_NOT] = "_UNARY_NOT", + [_UNPACK_EX] = "_UNPACK_EX", + [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", + [_UNPACK_SEQUENCE_LIST] = "_UNPACK_SEQUENCE_LIST", + [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", + [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", + [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", +}; +#endif // NEED_OPCODE_METADATA + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CORE_UOP_METADATA_H */ diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index e2b94894681f44..153884f4bd2902 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -24,7 +24,7 @@ typedef struct { _PyUOpInstruction trace[1]; } _PyUOpExecutorObject; -_PyInterpreterFrame *_PyUopExecute( +_Py_CODEUNIT *_PyUOpExecute( _PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer); diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index eacbe14c903289..dea267b49039e7 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -9,48 +9,66 @@ extern "C" { #endif #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_object.h" // _Py_REF_IS_MERGED() -static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) { +static inline int _is_dead(PyObject *obj) +{ + // Explanation for the Py_REFCNT() check: when a weakref's target is part + // of a long chain of deallocations which triggers the trashcan mechanism, + // clearing the weakrefs can be delayed long after the target's refcount + // has dropped to zero. In the meantime, code accessing the weakref will + // be able to "see" the target object even though it is supposed to be + // unreachable. See issue gh-60806. +#if defined(Py_GIL_DISABLED) + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared); + return shared == _Py_REF_SHARED(0, _Py_REF_MERGED); +#else + return (Py_REFCNT(obj) == 0); +#endif +} + +static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) +{ assert(PyWeakref_Check(ref_obj)); + PyObject *ret = NULL; + Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); PyObject *obj = ref->wr_object; if (obj == Py_None) { // clear_weakref() was called - return NULL; + goto end; } - // Explanation for the Py_REFCNT() check: when a weakref's target is part - // of a long chain of deallocations which triggers the trashcan mechanism, - // clearing the weakrefs can be delayed long after the target's refcount - // has dropped to zero. In the meantime, code accessing the weakref will - // be able to "see" the target object even though it is supposed to be - // unreachable. See issue gh-60806. - Py_ssize_t refcnt = Py_REFCNT(obj); - if (refcnt == 0) { - return NULL; + if (_is_dead(obj)) { + goto end; } - - assert(refcnt > 0); - return Py_NewRef(obj); +#if !defined(Py_GIL_DISABLED) + assert(Py_REFCNT(obj) > 0); +#endif + ret = Py_NewRef(obj); +end: + Py_END_CRITICAL_SECTION(); + return ret; } -static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) { +static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) +{ assert(PyWeakref_Check(ref_obj)); - int is_dead; + int ret = 0; Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); PyObject *obj = ref->wr_object; if (obj == Py_None) { // clear_weakref() was called - is_dead = 1; + ret = 1; } else { // See _PyWeakref_GET_REF() for the rationale of this test - is_dead = (Py_REFCNT(obj) == 0); + ret = _is_dead(obj); } Py_END_CRITICAL_SECTION(); - return is_dead; + return ret; } extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyWeakReference *head); diff --git a/Include/longobject.h b/Include/longobject.h index 7393254cd24a9b..51005efff636fa 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -7,7 +7,7 @@ extern "C" { /* Long (arbitrary precision) integer object interface */ -PyAPI_DATA(PyTypeObject) PyLong_Type; +// PyLong_Type is declared by object.h #define PyLong_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS) diff --git a/Include/modsupport.h b/Include/modsupport.h index 450cc99c404c17..ea4c0fce9f4562 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -9,10 +9,10 @@ extern "C" { PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...); PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...); PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, - const char *, char **, ...); + const char *, PY_CXX_CONST char * const *, ...); PyAPI_FUNC(int) PyArg_VaParse(PyObject *, const char *, va_list); PyAPI_FUNC(int) PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, - const char *, char **, va_list); + const char *, PY_CXX_CONST char * const *, va_list); PyAPI_FUNC(int) PyArg_ValidateKeywordArguments(PyObject *); PyAPI_FUNC(int) PyArg_UnpackTuple(PyObject *, const char *, Py_ssize_t, Py_ssize_t, ...); diff --git a/Include/object.h b/Include/object.h index 6b70a494844476..48f1ddf7510887 100644 --- a/Include/object.h +++ b/Include/object.h @@ -88,7 +88,7 @@ having all the lower 32 bits set, which will avoid the reference count to go beyond the refcount limit. Immortality checks for reference count decreases will be done by checking the bit sign flag in the lower 32 bits. */ -#define _Py_IMMORTAL_REFCNT UINT_MAX +#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX) #else /* @@ -103,7 +103,7 @@ immortality, but the execution would still be correct. Reference count increases and decreases will first go through an immortality check by comparing the reference count field to the immortality reference count. */ -#define _Py_IMMORTAL_REFCNT (UINT_MAX >> 2) +#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX >> 2) #endif // Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is @@ -239,6 +239,8 @@ PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); #define Py_Is(x, y) ((x) == (y)) #if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +PyAPI_FUNC(uintptr_t) _Py_GetThreadLocal_Addr(void); + static inline uintptr_t _Py_ThreadId(void) { @@ -261,8 +263,39 @@ _Py_ThreadId(void) __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); #elif defined(__aarch64__) __asm__ ("mrs %0, tpidr_el0" : "=r" (tid)); +#elif defined(__powerpc64__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r13 is reserved for use as system thread ID by the Power 64-bit ABI. + register uintptr_t tp __asm__ ("r13"); + __asm__("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__powerpc__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r2 is reserved for use as system thread ID by the Power 32-bit ABI. + register uintptr_t tp __asm__ ("r2"); + __asm__ ("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__s390__) && defined(__GNUC__) + // Both GCC and Clang have supported __builtin_thread_pointer + // for s390 from long time ago. + tid = (uintptr_t)__builtin_thread_pointer(); +#elif defined(__riscv) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // tp is Thread Pointer provided by the RISC-V ABI. + __asm__ ("mv %0, tp" : "=r" (tid)); + #endif #else - # error "define _Py_ThreadId for this platform" + // Fallback to a portable implementation if we do not have a faster + // platform-specific implementation. + tid = _Py_GetThreadLocal_Addr(); #endif return tid; } @@ -317,11 +350,11 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) { static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) { #if defined(Py_GIL_DISABLED) - return op->ob_ref_local == _Py_IMMORTAL_REFCNT_LOCAL; + return (op->ob_ref_local == _Py_IMMORTAL_REFCNT_LOCAL); #elif SIZEOF_VOID_P > 4 - return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; + return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0); #else - return op->ob_refcnt == _Py_IMMORTAL_REFCNT; + return (op->ob_refcnt == _Py_IMMORTAL_REFCNT); #endif } #define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) @@ -350,15 +383,23 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { if (_Py_IsImmortal(ob)) { return; } + #ifndef Py_GIL_DISABLED ob->ob_refcnt = refcnt; #else if (_Py_IsOwnedByCurrentThread(ob)) { - // Set local refcount to desired refcount and shared refcount to zero, - // but preserve the shared refcount flags. - assert(refcnt < UINT32_MAX); - ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); - ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; + if ((size_t)refcnt > (size_t)UINT32_MAX) { + // On overflow, make the object immortal + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + ob->ob_ref_shared = 0; + } + else { + // Set local refcount to desired refcount and shared refcount + // to zero, but preserve the shared refcount flags. + ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); + ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; + } } else { // Set local refcount to zero and shared refcount to desired refcount. @@ -750,6 +791,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); uint32_t new_local = local + 1; if (new_local == 0) { + // local is equal to _Py_IMMORTAL_REFCNT: do nothing return; } if (_Py_IsOwnedByCurrentThread(op)) { @@ -763,6 +805,8 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; PY_UINT32_T new_refcnt = cur_refcnt + 1; if (new_refcnt == 0) { + // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal, + // do nothing return; } op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; diff --git a/Include/objimpl.h b/Include/objimpl.h index 967e2776767756..ff5fa7a8c1d3d8 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -35,7 +35,7 @@ Functions and macros for modules that implement new object types. fields, this also fills in the ob_size field. - PyObject_Free(op) releases the memory allocated for an object. It does not - run a destructor -- it only frees the memory. PyObject_Free is identical. + run a destructor -- it only frees the memory. - PyObject_Init(op, typeobj) and PyObject_InitVar(op, typeobj, n) don't allocate memory. Instead of a 'type' parameter, they take a pointer to a diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index ba25bd459c1bcd..fe969342ee79e7 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -1,4 +1,4 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/opcode_id_generator.py // from: // Python/bytecodes.c // Do not edit! @@ -55,7 +55,6 @@ extern "C" { #define UNARY_NEGATIVE 42 #define UNARY_NOT 43 #define WITH_EXCEPT_START 44 -#define HAVE_ARGUMENT 45 #define BINARY_OP 45 #define BUILD_CONST_KEY_MAP 46 #define BUILD_LIST 47 @@ -200,7 +199,6 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 216 #define UNPACK_SEQUENCE_TUPLE 217 #define UNPACK_SEQUENCE_TWO_TUPLE 218 -#define MIN_INSTRUMENTED_OPCODE 236 #define INSTRUMENTED_RESUME 236 #define INSTRUMENTED_END_FOR 237 #define INSTRUMENTED_END_SEND 238 @@ -233,6 +231,9 @@ extern "C" { #define SETUP_WITH 266 #define STORE_FAST_MAYBE_NULL 267 +#define HAVE_ARGUMENT 44 +#define MIN_INSTRUMENTED_OPCODE 236 + #ifdef __cplusplus } #endif diff --git a/Include/pymacconfig.h b/Include/pymacconfig.h index 806e41955efd7f..615abe103ca038 100644 --- a/Include/pymacconfig.h +++ b/Include/pymacconfig.h @@ -7,7 +7,9 @@ #define PY_MACCONFIG_H #ifdef __APPLE__ +#undef ALIGNOF_MAX_ALIGN_T #undef SIZEOF_LONG +#undef SIZEOF_LONG_DOUBLE #undef SIZEOF_PTHREAD_T #undef SIZEOF_SIZE_T #undef SIZEOF_TIME_T @@ -20,6 +22,7 @@ #undef DOUBLE_IS_BIG_ENDIAN_IEEE754 #undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 #undef HAVE_GCC_ASM_FOR_X87 +#undef HAVE_GCC_ASM_FOR_X64 #undef VA_LIST_IS_ARRAY #if defined(__LP64__) && defined(__x86_64__) @@ -74,8 +77,14 @@ # define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 #endif -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) # define HAVE_GCC_ASM_FOR_X87 +# define ALIGNOF_MAX_ALIGN_T 16 +# define HAVE_GCC_ASM_FOR_X64 1 +# define SIZEOF_LONG_DOUBLE 16 +#else +# define ALIGNOF_MAX_ALIGN_T 8 +# define SIZEOF_LONG_DOUBLE 8 #endif #endif // __APPLE__ diff --git a/Include/pyport.h b/Include/pyport.h index abb526d503fddd..9d7ef0061806ad 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -563,6 +563,11 @@ extern "C" { # define _Py_ADDRESS_SANITIZER # endif # endif +# if __has_feature(thread_sanitizer) +# if !defined(_Py_THREAD_SANITIZER) +# define _Py_THREAD_SANITIZER +# endif +# endif #elif defined(__GNUC__) # if defined(__SANITIZE_ADDRESS__) # define _Py_ADDRESS_SANITIZER @@ -586,6 +591,14 @@ extern "C" { # define ALIGNOF_MAX_ALIGN_T _Alignof(long double) #endif +#ifndef PY_CXX_CONST +# ifdef __cplusplus +# define PY_CXX_CONST const +# else +# define PY_CXX_CONST +# endif +#endif + #if defined(__sgi) && !defined(_SGI_MP_SOURCE) # define _SGI_MP_SOURCE #endif diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 5dd06ae487dfcf..fdb099bd0c2ecf 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -1,8 +1,7 @@ -# This file is generated by Tools/cases_generator/generate_cases.py +# This file is generated by Tools/cases_generator/py_metadata_generator.py # from: # Python/bytecodes.c # Do not edit! - _specializations = { "RESUME": [ "RESUME_CHECK", @@ -23,6 +22,7 @@ "BINARY_OP_ADD_FLOAT", "BINARY_OP_SUBTRACT_FLOAT", "BINARY_OP_ADD_UNICODE", + "BINARY_OP_INPLACE_ADD_UNICODE", ], "BINARY_SUBSCR": [ "BINARY_SUBSCR_DICT", @@ -103,14 +103,11 @@ ], } -# An irregular case: -_specializations["BINARY_OP"].append("BINARY_OP_INPLACE_ADD_UNICODE") - _specialized_opmap = { - 'BINARY_OP_INPLACE_ADD_UNICODE': 3, 'BINARY_OP_ADD_FLOAT': 150, 'BINARY_OP_ADD_INT': 151, 'BINARY_OP_ADD_UNICODE': 152, + 'BINARY_OP_INPLACE_ADD_UNICODE': 3, 'BINARY_OP_MULTIPLY_FLOAT': 153, 'BINARY_OP_MULTIPLY_INT': 154, 'BINARY_OP_SUBTRACT_FLOAT': 155, @@ -181,6 +178,9 @@ opmap = { 'CACHE': 0, + 'RESERVED': 17, + 'RESUME': 149, + 'INSTRUMENTED_LINE': 254, 'BEFORE_ASYNC_WITH': 1, 'BEFORE_WITH': 2, 'BINARY_SLICE': 4, @@ -196,7 +196,6 @@ 'FORMAT_SIMPLE': 14, 'FORMAT_WITH_SPEC': 15, 'GET_AITER': 16, - 'RESERVED': 17, 'GET_ANEXT': 18, 'GET_ITER': 19, 'GET_LEN': 20, @@ -298,7 +297,6 @@ 'UNPACK_EX': 116, 'UNPACK_SEQUENCE': 117, 'YIELD_VALUE': 118, - 'RESUME': 149, 'INSTRUMENTED_RESUME': 236, 'INSTRUMENTED_END_FOR': 237, 'INSTRUMENTED_END_SEND': 238, @@ -317,7 +315,6 @@ 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, - 'INSTRUMENTED_LINE': 254, 'JUMP': 256, 'JUMP_NO_INTERRUPT': 257, 'LOAD_CLOSURE': 258, @@ -331,5 +328,6 @@ 'SETUP_WITH': 266, 'STORE_FAST_MAYBE_NULL': 267, } + +HAVE_ARGUMENT = 44 MIN_INSTRUMENTED_OPCODE = 236 -HAVE_ARGUMENT = 45 diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index aa66c8b9f4189f..0cb064fcd791be 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -507,6 +507,11 @@ def get_platform_osx(_config_vars, osname, release, machine): # MACOSX_DEPLOYMENT_TARGET. macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '') + if macver and '.' not in macver: + # Ensure that the version includes at least a major + # and minor version, even if MACOSX_DEPLOYMENT_TARGET + # is set to a single-label version like "14". + macver += '.0' macrelease = _get_system_version() or macver macver = macver or macrelease diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 77ccdc9e1d789b..798cf9f9d3fffe 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -342,8 +342,6 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): tz = -1 gmtoff = None gmtoff_fraction = 0 - # Default to -1 to signify that values not known; not critical to have, - # though iso_week = week_of_year = None week_of_year_start = None # weekday and julian defaulted to None so as to signal need to calculate @@ -470,17 +468,17 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # Deal with the cases where ambiguities arise # don't assume default values for ISO week/year - if year is None and iso_year is not None: - if iso_week is None or weekday is None: - raise ValueError("ISO year directive '%G' must be used with " - "the ISO week directive '%V' and a weekday " - "directive ('%A', '%a', '%w', or '%u').") + if iso_year is not None: if julian is not None: raise ValueError("Day of the year directive '%j' is not " "compatible with ISO year directive '%G'. " "Use '%Y' instead.") - elif week_of_year is None and iso_week is not None: - if weekday is None: + elif iso_week is None or weekday is None: + raise ValueError("ISO year directive '%G' must be used with " + "the ISO week directive '%V' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + elif iso_week is not None: + if year is None or weekday is None: raise ValueError("ISO week directive '%V' must be used with " "the ISO year directive '%G' and a weekday " "directive ('%A', '%a', '%w', or '%u').") @@ -490,11 +488,12 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): "instead.") leap_year_fix = False - if year is None and month == 2 and day == 29: - year = 1904 # 1904 is first leap year of 20th century - leap_year_fix = True - elif year is None: - year = 1900 + if year is None: + if month == 2 and day == 29: + year = 1904 # 1904 is first leap year of 20th century + leap_year_fix = True + else: + year = 1900 # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 416c732298d9a9..a8870b636d1df5 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -1496,6 +1496,7 @@ async def create_server( ssl=None, reuse_address=None, reuse_port=None, + keep_alive=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=True): @@ -1569,6 +1570,9 @@ async def create_server( socket.SOL_SOCKET, socket.SO_REUSEADDR, True) if reuse_port: _set_reuseport(sock) + if keep_alive: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) # Disable IPv4/IPv6 dual stack support (enabled by # default on Linux) which makes a single socket # listen on both address families. diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 4c9b0dd5653c0c..6dbde2b696ad1f 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -115,7 +115,8 @@ def close(self): try: self._proc.kill() - except ProcessLookupError: + except (ProcessLookupError, PermissionError): + # the process may have already exited or may be running setuid pass # Don't clear the _proc reference yet: _post_init() may still run diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0ccf85105e6673..ebc3836bdc0c4d 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -316,6 +316,7 @@ async def create_server( *, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None, + keep_alive=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, start_serving=True): @@ -354,6 +355,9 @@ async def create_server( they all set this flag when being created. This option is not supported on Windows. + keep_alive set to True keeps connections active by enabling the + periodic transmission of messages. + ssl_handshake_timeout is the time in seconds that an SSL server will wait for completion of the SSL handshake before aborting the connection. Default is 60s. diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index d521b4e2e255a9..dcd5e0aa345029 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -261,15 +261,11 @@ def _ensure_fd_no_transport(self, fd): except (AttributeError, TypeError, ValueError): # This code matches selectors._fileobj_to_fd function. raise ValueError(f"Invalid file object: {fd!r}") from None - try: - transport = self._transports[fileno] - except KeyError: - pass - else: - if not transport.is_closing(): - raise RuntimeError( - f'File descriptor {fd!r} is used by transport ' - f'{transport!r}') + transport = self._transports.get(fileno) + if transport and not transport.is_closing(): + raise RuntimeError( + f'File descriptor {fd!r} is used by transport ' + f'{transport!r}') def _add_reader(self, fd, callback, *args): self._check_closed() diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 3eb65a8a08b5a0..cbb6527d0b28e0 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -243,13 +243,12 @@ def abort(self): The protocol's connection_lost() method will (eventually) be called with None as its argument. """ - self._closed = True - if self._ssl_protocol is not None: - self._ssl_protocol._abort() + self._force_close(None) def _force_close(self, exc): self._closed = True - self._ssl_protocol._abort(exc) + if self._ssl_protocol is not None: + self._ssl_protocol._abort(exc) def _test__append_write_backlog(self, data): # for test only @@ -614,7 +613,7 @@ def _start_shutdown(self): if self._app_transport is not None: self._app_transport._closed = True if self._state == SSLProtocolState.DO_HANDSHAKE: - self._abort() + self._abort(None) else: self._set_state(SSLProtocolState.FLUSHING) self._shutdown_timeout_handle = self._loop.call_later( @@ -661,10 +660,10 @@ def _on_shutdown_complete(self, shutdown_exc): else: self._loop.call_soon(self._transport.close) - def _abort(self): + def _abort(self, exc): self._set_state(SSLProtocolState.UNWRAPPED) if self._transport is not None: - self._transport.abort() + self._transport._force_close(exc) # Outgoing flow diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 91be0decc41c42..cb9c1ce4d7d1d2 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -158,10 +158,10 @@ def create_task(self, coro, *, name=None, context=None): if self._aborting: raise RuntimeError(f"TaskGroup {self!r} is shutting down") if context is None: - task = self._loop.create_task(coro) + task = self._loop.create_task(coro, name=name) else: - task = self._loop.create_task(coro, context=context) - task.set_name(name) + task = self._loop.create_task(coro, name=name, context=context) + # optimization: Immediately call the done callback if the task is # already done (e.g. if the coro was able to complete eagerly), # and skip scheduling a done callback diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index e84b21390557be..fafee3e738f6aa 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -404,11 +404,10 @@ def create_task(coro, *, name=None, context=None): loop = events.get_running_loop() if context is None: # Use legacy API if context is not needed - task = loop.create_task(coro) + task = loop.create_task(coro, name=name) else: - task = loop.create_task(coro, context=context) + task = loop.create_task(coro, name=name, context=context) - task.set_name(name) return task diff --git a/Lib/cmd.py b/Lib/cmd.py index e933b8dbc1470a..a37d16cd7bde16 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -42,7 +42,7 @@ functions respectively. """ -import string, sys +import inspect, string, sys __all__ = ["Cmd"] @@ -108,7 +108,15 @@ def cmdloop(self, intro=None): import readline self.old_completer = readline.get_completer() readline.set_completer(self.complete) - readline.parse_and_bind(self.completekey+": complete") + if readline.backend == "editline": + if self.completekey == 'tab': + # libedit uses "^I" instead of "tab" + command_string = "bind ^I rl_complete" + else: + command_string = f"bind {self.completekey} rl_complete" + else: + command_string = f"{self.completekey}: complete" + readline.parse_and_bind(command_string) except ImportError: pass try: @@ -297,6 +305,7 @@ def do_help(self, arg): except AttributeError: try: doc=getattr(self, 'do_' + arg).__doc__ + doc = inspect.cleandoc(doc) if doc: self.stdout.write("%s\n"%str(doc)) return diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index a461550ea40da7..2e527dfd810c43 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -457,7 +457,7 @@ def _make(cls, iterable): def _replace(self, /, **kwds): result = self._make(_map(kwds.pop, field_names, self)) if kwds: - raise ValueError(f'Got unexpected field names: {list(kwds)!r}') + raise TypeError(f'Got unexpected field names: {list(kwds)!r}') return result _replace.__doc__ = (f'Return a new {typename} object replacing specified ' diff --git a/Lib/dis.py b/Lib/dis.py index e08e9a94057689..1a2f1032d500af 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -113,7 +113,14 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, elif hasattr(x, 'co_code'): # Code object _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) elif isinstance(x, (bytes, bytearray)): # Raw bytecode - _disassemble_bytes(x, file=file, show_caches=show_caches, show_offsets=show_offsets) + labels_map = _make_labels_map(x) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=file, + offset_width=len(str(max(len(x) - 2, 9999))) if show_offsets else 0, + label_width=label_width, + show_caches=show_caches) + arg_resolver = ArgResolver(labels_map=labels_map) + _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) elif isinstance(x, str): # Source code _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) else: @@ -267,9 +274,10 @@ def show_code(co, *, file=None): 'starts_line', 'line_number', 'label', - 'positions' + 'positions', + 'cache_info', ], - defaults=[None, None] + defaults=[None, None, None] ) _Instruction.opname.__doc__ = "Human readable name for operation" @@ -286,6 +294,7 @@ def show_code(co, *, file=None): _Instruction.line_number.__doc__ = "source line number associated with this opcode (if any), otherwise None" _Instruction.label.__doc__ = "A label (int > 0) if this instruction is a jump target, otherwise None" _Instruction.positions.__doc__ = "dis.Positions object holding the span of source code covered by this instruction" +_Instruction.cache_info.__doc__ = "list of (name, size, data), one for each cache entry of the instruction" _ExceptionTableEntryBase = collections.namedtuple("_ExceptionTableEntryBase", "start end target depth lasti") @@ -334,96 +343,10 @@ class Instruction(_Instruction): label - A label if this instruction is a jump target, otherwise None positions - Optional dis.Positions object holding the span of source code covered by this instruction + cache_info - information about the format and content of the instruction's cache + entries (if any) """ - @staticmethod - def _get_argval_argrepr(op, arg, offset, co_consts, names, varname_from_oparg, - labels_map): - get_name = None if names is None else names.__getitem__ - argval = None - argrepr = '' - deop = _deoptop(op) - if arg is not None: - # Set argval to the dereferenced value of the argument when - # available, and argrepr to the string representation of argval. - # _disassemble_bytes needs the string repr of the - # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. - argval = arg - if deop in hasconst: - argval, argrepr = _get_const_info(deop, arg, co_consts) - elif deop in hasname: - if deop == LOAD_GLOBAL: - argval, argrepr = _get_name_info(arg//2, get_name) - if (arg & 1) and argrepr: - argrepr = f"{argrepr} + NULL" - elif deop == LOAD_ATTR: - argval, argrepr = _get_name_info(arg//2, get_name) - if (arg & 1) and argrepr: - argrepr = f"{argrepr} + NULL|self" - elif deop == LOAD_SUPER_ATTR: - argval, argrepr = _get_name_info(arg//4, get_name) - if (arg & 1) and argrepr: - argrepr = f"{argrepr} + NULL|self" - else: - argval, argrepr = _get_name_info(arg, get_name) - elif deop in hasjabs: - argval = arg*2 - argrepr = f"to L{labels_map[argval]}" - elif deop in hasjrel: - signed_arg = -arg if _is_backward_jump(deop) else arg - argval = offset + 2 + signed_arg*2 - caches = _get_cache_size(_all_opname[deop]) - argval += 2 * caches - if deop == ENTER_EXECUTOR: - argval += 2 - argrepr = f"to L{labels_map[argval]}" - elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): - arg1 = arg >> 4 - arg2 = arg & 15 - val1, argrepr1 = _get_name_info(arg1, varname_from_oparg) - val2, argrepr2 = _get_name_info(arg2, varname_from_oparg) - argrepr = argrepr1 + ", " + argrepr2 - argval = val1, val2 - elif deop in haslocal or deop in hasfree: - argval, argrepr = _get_name_info(arg, varname_from_oparg) - elif deop in hascompare: - argval = cmp_op[arg >> 5] - argrepr = argval - if arg & 16: - argrepr = f"bool({argrepr})" - elif deop == CONVERT_VALUE: - argval = (None, str, repr, ascii)[arg] - argrepr = ('', 'str', 'repr', 'ascii')[arg] - elif deop == SET_FUNCTION_ATTRIBUTE: - argrepr = ', '.join(s for i, s in enumerate(FUNCTION_ATTR_FLAGS) - if arg & (1<' marker arrow as part of the line *offset_width* sets the width of the instruction offset field + *label_width* sets the width of the label field + *show_caches* is a boolean indicating whether to display cache lines + """ + self.file = file + self.lineno_width = lineno_width + self.offset_width = offset_width + self.label_width = label_width + self.show_caches = show_caches + + def print_instruction(self, instr, mark_as_current=False): + self.print_instruction_line(instr, mark_as_current) + if self.show_caches and instr.cache_info: + offset = instr.offset + for name, size, data in instr.cache_info: + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + self.print_instruction_line( + Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, + False, None, None, instr.positions), + False) + + def print_instruction_line(self, instr, mark_as_current): + """Format instruction details for inclusion in disassembly output.""" + lineno_width = self.lineno_width + offset_width = self.offset_width + label_width = self.label_width + + new_source_line = (lineno_width > 0 and + instr.starts_line and + instr.offset > 0) + if new_source_line: + print(file=self.file) + fields = [] # Column: Source code line number if lineno_width: - if self.starts_line: - lineno_fmt = "%%%dd" if self.line_number is not None else "%%%ds" + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" lineno_fmt = lineno_fmt % lineno_width - lineno = self.line_number if self.line_number is not None else '--' + lineno = _NO_LINENO if instr.line_number is None else instr.line_number fields.append(lineno_fmt % lineno) else: fields.append(' ' * lineno_width) # Column: Label - if self.label is not None: - lbl = f"L{self.label}:" - fields.append(f"{lbl:>{self.label_width}}") + if instr.label is not None: + lbl = f"L{instr.label}:" + fields.append(f"{lbl:>{label_width}}") else: - fields.append(' ' * self.label_width) + fields.append(' ' * label_width) # Column: Instruction offset from start of code sequence if offset_width > 0: - fields.append(f"{repr(self.offset):>{offset_width}} ") + fields.append(f"{repr(instr.offset):>{offset_width}} ") # Column: Current instruction indicator if mark_as_current: fields.append('-->') else: fields.append(' ') # Column: Opcode name - fields.append(self.opname.ljust(_OPNAME_WIDTH)) + fields.append(instr.opname.ljust(_OPNAME_WIDTH)) # Column: Opcode argument - if self.arg is not None: - arg = repr(self.arg) + if instr.arg is not None: + arg = repr(instr.arg) # If opname is longer than _OPNAME_WIDTH, we allow it to overflow into # the space reserved for oparg. This results in fewer misaligned opargs # in the disassembly output. - opname_excess = max(0, len(self.opname) - _OPNAME_WIDTH) - fields.append(repr(self.arg).rjust(_OPARG_WIDTH - opname_excess)) + opname_excess = max(0, len(instr.opname) - _OPNAME_WIDTH) + fields.append(repr(instr.arg).rjust(_OPARG_WIDTH - opname_excess)) # Column: Opcode argument details - if self.argrepr: - fields.append('(' + self.argrepr + ')') - return ' '.join(fields).rstrip() - - def __str__(self): - return self._disassemble() + if instr.argrepr: + fields.append('(' + instr.argrepr + ')') + print(' '.join(fields).rstrip(), file=self.file) + + def print_exception_table(self, exception_entries): + file = self.file + if exception_entries: + print("ExceptionTable:", file=file) + for entry in exception_entries: + lasti = " lasti" if entry.lasti else "" + start = entry.start_label + end = entry.end_label + target = entry.target_label + print(f" L{start} to L{end} -> L{target} [{entry.depth}]{lasti}", file=file) + + +class ArgResolver: + def __init__(self, co_consts=None, names=None, varname_from_oparg=None, labels_map=None): + self.co_consts = co_consts + self.names = names + self.varname_from_oparg = varname_from_oparg + self.labels_map = labels_map or {} + + def get_label_for_offset(self, offset): + return self.labels_map.get(offset, None) + + def get_argval_argrepr(self, op, arg, offset): + get_name = None if self.names is None else self.names.__getitem__ + argval = None + argrepr = '' + deop = _deoptop(op) + if arg is not None: + # Set argval to the dereferenced value of the argument when + # available, and argrepr to the string representation of argval. + # _disassemble_bytes needs the string repr of the + # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. + argval = arg + if deop in hasconst: + argval, argrepr = _get_const_info(deop, arg, self.co_consts) + elif deop in hasname: + if deop == LOAD_GLOBAL: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL" + elif deop == LOAD_ATTR: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + elif deop == LOAD_SUPER_ATTR: + argval, argrepr = _get_name_info(arg//4, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + else: + argval, argrepr = _get_name_info(arg, get_name) + elif deop in hasjabs: + argval = arg*2 + argrepr = f"to L{self.labels_map[argval]}" + elif deop in hasjrel: + signed_arg = -arg if _is_backward_jump(deop) else arg + argval = offset + 2 + signed_arg*2 + caches = _get_cache_size(_all_opname[deop]) + argval += 2 * caches + if deop == ENTER_EXECUTOR: + argval += 2 + argrepr = f"to L{self.labels_map[argval]}" + elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): + arg1 = arg >> 4 + arg2 = arg & 15 + val1, argrepr1 = _get_name_info(arg1, self.varname_from_oparg) + val2, argrepr2 = _get_name_info(arg2, self.varname_from_oparg) + argrepr = argrepr1 + ", " + argrepr2 + argval = val1, val2 + elif deop in haslocal or deop in hasfree: + argval, argrepr = _get_name_info(arg, self.varname_from_oparg) + elif deop in hascompare: + argval = cmp_op[arg >> 5] + argrepr = argval + if arg & 16: + argrepr = f"bool({argrepr})" + elif deop == CONVERT_VALUE: + argval = (None, str, repr, ascii)[arg] + argrepr = ('', 'str', 'repr', 'ascii')[arg] + elif deop == SET_FUNCTION_ATTRIBUTE: + argrepr = ', '.join(s for i, s in enumerate(FUNCTION_ATTR_FLAGS) + if arg & (1< 0 - else: - show_lineno = False - - if show_lineno: - maxlineno = max(linestarts_ints) + line_offset - if maxlineno >= 1000: - lineno_width = len(str(maxlineno)) - else: - lineno_width = 3 - if lineno_width < len(str(None)) and None in linestarts.values(): - lineno_width = len(str(None)) - else: - lineno_width = 0 - if show_offsets: - maxoffset = len(code) - 2 - if maxoffset >= 10000: - offset_width = len(str(maxoffset)) - else: - offset_width = 4 - else: - offset_width = 0 - - for instr in _get_instructions_bytes(code, varname_from_oparg, names, - co_consts, linestarts, - line_offset=line_offset, - exception_entries=exception_entries, - co_positions=co_positions, - show_caches=show_caches, - original_code=original_code): - new_source_line = (show_lineno and - instr.starts_line and - instr.offset > 0) - if new_source_line: - print(file=file) - if show_caches: - is_current_instr = instr.offset == lasti - else: - # Each CACHE takes 2 bytes - is_current_instr = instr.offset <= lasti \ - <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) - print(instr._disassemble(lineno_width, is_current_instr, offset_width), - file=file) - if exception_entries: - print("ExceptionTable:", file=file) - for entry in exception_entries: - lasti = " lasti" if entry.lasti else "" - start = entry.start_label - end = entry.end_label - target = entry.target_label - print(f" L{start} to L{end} -> L{target} [{entry.depth}]{lasti}", file=file) +def _make_labels_map(original_code, exception_entries=()): + jump_targets = set(findlabels(original_code)) + labels = set(jump_targets) + for start, end, target, _, _ in exception_entries: + labels.add(start) + labels.add(end) + labels.add(target) + labels = sorted(labels) + labels_map = {offset: i+1 for (i, offset) in enumerate(sorted(labels))} + for e in exception_entries: + e.start_label = labels_map[e.start] + e.end_label = labels_map[e.end] + e.target_label = labels_map[e.target] + return labels_map + +_NO_LINENO = ' --' + +def _get_lineno_width(linestarts): + if linestarts is None: + return 0 + maxlineno = max(filter(None, linestarts.values()), default=-1) + if maxlineno == -1: + # Omit the line number column entirely if we have no line number info + return 0 + lineno_width = max(3, len(str(maxlineno))) + if lineno_width < len(_NO_LINENO) and None in linestarts.values(): + lineno_width = len(_NO_LINENO) + return lineno_width + + +def _disassemble_bytes(code, lasti=-1, linestarts=None, + *, line_offset=0, exception_entries=(), + co_positions=None, original_code=None, + arg_resolver=None, formatter=None): + + assert formatter is not None + assert arg_resolver is not None + + instrs = _get_instructions_bytes(code, linestarts=linestarts, + line_offset=line_offset, + co_positions=co_positions, + original_code=original_code, + arg_resolver=arg_resolver) + + print_instructions(instrs, exception_entries, formatter, lasti=lasti) + + +def print_instructions(instrs, exception_entries, formatter, lasti=-1): + for instr in instrs: + # Each CACHE takes 2 bytes + is_current_instr = instr.offset <= lasti \ + <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) + formatter.print_instruction(instr, is_current_instr) + + formatter.print_exception_table(exception_entries) def _disassemble_str(source, **kwargs): """Compile the source string, then disassemble the code object.""" @@ -922,15 +957,18 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False def __iter__(self): co = self.codeobj + original_code = co.co_code + labels_map = _make_labels_map(original_code, self.exception_entries) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) return _get_instructions_bytes(_get_code_array(co, self.adaptive), - co._varname_from_oparg, - co.co_names, co.co_consts, - self._linestarts, + linestarts=self._linestarts, line_offset=self._line_offset, - exception_entries=self.exception_entries, co_positions=co.co_positions(), - show_caches=self.show_caches, - original_code=co.co_code) + original_code=original_code, + arg_resolver=arg_resolver) def __repr__(self): return "{}({!r})".format(self.__class__.__name__, @@ -957,18 +995,32 @@ def dis(self): else: offset = -1 with io.StringIO() as output: - _disassemble_bytes(_get_code_array(co, self.adaptive), - varname_from_oparg=co._varname_from_oparg, - names=co.co_names, co_consts=co.co_consts, + code = _get_code_array(co, self.adaptive) + offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 + + + labels_map = _make_labels_map(co.co_code, self.exception_entries) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=output, + lineno_width=_get_lineno_width(self._linestarts), + offset_width=offset_width, + label_width=label_width, + line_offset=self._line_offset, + show_caches=self.show_caches) + + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + _disassemble_bytes(code, linestarts=self._linestarts, line_offset=self._line_offset, - file=output, lasti=offset, exception_entries=self.exception_entries, co_positions=co.co_positions(), - show_caches=self.show_caches, original_code=co.co_code, - show_offsets=self.show_offsets) + arg_resolver=arg_resolver, + formatter=formatter) return output.getvalue() diff --git a/Lib/doctest.py b/Lib/doctest.py index d109b6c9e37343..114aac62a34e95 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1136,6 +1136,8 @@ def _find_lineno(self, obj, source_lines): # Find the line number for functions & methods. if inspect.ismethod(obj): obj = obj.__func__ + if isinstance(obj, property): + obj = obj.fget if inspect.isfunction(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. obj = obj.__code__ diff --git a/Lib/email/message.py b/Lib/email/message.py index 411118c74dabb4..fe769580fed5d0 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -289,25 +289,26 @@ def get_payload(self, i=None, decode=False): # cte might be a Header, so for now stringify it. cte = str(self.get('content-transfer-encoding', '')).lower() # payload may be bytes here. - if isinstance(payload, str): - if utils._has_surrogates(payload): - bpayload = payload.encode('ascii', 'surrogateescape') - if not decode: + if not decode: + if isinstance(payload, str) and utils._has_surrogates(payload): + try: + bpayload = payload.encode('ascii', 'surrogateescape') try: payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace') except LookupError: payload = bpayload.decode('ascii', 'replace') - elif decode: - try: - bpayload = payload.encode('ascii') - except UnicodeError: - # This won't happen for RFC compliant messages (messages - # containing only ASCII code points in the unicode input). - # If it does happen, turn the string into bytes in a way - # guaranteed not to fail. - bpayload = payload.encode('raw-unicode-escape') - if not decode: + except UnicodeEncodeError: + pass return payload + if isinstance(payload, str): + try: + bpayload = payload.encode('ascii', 'surrogateescape') + except UnicodeEncodeError: + # This won't happen for RFC compliant messages (messages + # containing only ASCII code points in the unicode input). + # If it does happen, turn the string into bytes in a way + # guaranteed not to fail. + bpayload = payload.encode('raw-unicode-escape') if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': diff --git a/Lib/email/utils.py b/Lib/email/utils.py index a49a8fa986ce0c..103cef61a83538 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -43,11 +43,12 @@ specialsre = re.compile(r'[][\\()<>@,:;".]') escapesre = re.compile(r'[\\"]') + def _has_surrogates(s): - """Return True if s contains surrogate-escaped binary data.""" + """Return True if s may contain surrogate-escaped binary data.""" # This check is based on the fact that unless there are surrogates, utf8 # (Python's default encoding) can encode any string. This is the fastest - # way to check for surrogates, see issue 11454 for timings. + # way to check for surrogates, see bpo-11454 (moved to gh-55663) for timings. try: s.encode() return False @@ -103,12 +104,127 @@ def formataddr(pair, charset='utf-8'): return address +def _iter_escaped_chars(addr): + pos = 0 + escape = False + for pos, ch in enumerate(addr): + if escape: + yield (pos, '\\' + ch) + escape = False + elif ch == '\\': + escape = True + else: + yield (pos, ch) + if escape: + yield (pos, '\\') + + +def _strip_quoted_realnames(addr): + """Strip real names between quotes.""" + if '"' not in addr: + # Fast path + return addr + + start = 0 + open_pos = None + result = [] + for pos, ch in _iter_escaped_chars(addr): + if ch == '"': + if open_pos is None: + open_pos = pos + else: + if start != open_pos: + result.append(addr[start:open_pos]) + start = pos + 1 + open_pos = None + + if start < len(addr): + result.append(addr[start:]) + + return ''.join(result) -def getaddresses(fieldvalues): - """Return a list of (REALNAME, EMAIL) for each fieldvalue.""" - all = COMMASPACE.join(str(v) for v in fieldvalues) - a = _AddressList(all) - return a.addresslist + +supports_strict_parsing = True + +def getaddresses(fieldvalues, *, strict=True): + """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue. + + When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in + its place. + + If strict is true, use a strict parser which rejects malformed inputs. + """ + + # If strict is true, if the resulting list of parsed addresses is greater + # than the number of fieldvalues in the input list, a parsing error has + # occurred and consequently a list containing a single empty 2-tuple [('', + # '')] is returned in its place. This is done to avoid invalid output. + # + # Malformed input: getaddresses(['alice@example.com ']) + # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')] + # Safe output: [('', '')] + + if not strict: + all = COMMASPACE.join(str(v) for v in fieldvalues) + a = _AddressList(all) + return a.addresslist + + fieldvalues = [str(v) for v in fieldvalues] + fieldvalues = _pre_parse_validation(fieldvalues) + addr = COMMASPACE.join(fieldvalues) + a = _AddressList(addr) + result = _post_parse_validation(a.addresslist) + + # Treat output as invalid if the number of addresses is not equal to the + # expected number of addresses. + n = 0 + for v in fieldvalues: + # When a comma is used in the Real Name part it is not a deliminator. + # So strip those out before counting the commas. + v = _strip_quoted_realnames(v) + # Expected number of addresses: 1 + number of commas + n += 1 + v.count(',') + if len(result) != n: + return [('', '')] + + return result + + +def _check_parenthesis(addr): + # Ignore parenthesis in quoted real names. + addr = _strip_quoted_realnames(addr) + + opens = 0 + for pos, ch in _iter_escaped_chars(addr): + if ch == '(': + opens += 1 + elif ch == ')': + opens -= 1 + if opens < 0: + return False + return (opens == 0) + + +def _pre_parse_validation(email_header_fields): + accepted_values = [] + for v in email_header_fields: + if not _check_parenthesis(v): + v = "('', '')" + accepted_values.append(v) + + return accepted_values + + +def _post_parse_validation(parsed_email_header_tuples): + accepted_values = [] + # The parser would have parsed a correctly formatted domain-literal + # The existence of an [ after parsing indicates a parsing failure + for v in parsed_email_header_tuples: + if '[' in v[1]: + v = ('', '') + accepted_values.append(v) + + return accepted_values def _format_timetuple_and_zone(timetuple, zone): @@ -207,16 +323,33 @@ def parsedate_to_datetime(data): tzinfo=datetime.timezone(datetime.timedelta(seconds=tz))) -def parseaddr(addr): +def parseaddr(addr, *, strict=True): """ Parse addr into its constituent realname and email address parts. Return a tuple of realname and email address, unless the parse fails, in which case return a 2-tuple of ('', ''). + + If strict is True, use a strict parser which rejects malformed inputs. """ - addrs = _AddressList(addr).addresslist - if not addrs: - return '', '' + if not strict: + addrs = _AddressList(addr).addresslist + if not addrs: + return ('', '') + return addrs[0] + + if isinstance(addr, list): + addr = addr[0] + + if not isinstance(addr, str): + return ('', '') + + addr = _pre_parse_validation([addr])[0] + addrs = _post_parse_validation(_AddressList(addr).addresslist) + + if not addrs or len(addrs) > 1: + return ('', '') + return addrs[0] diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 1fb1d505cfd0c5..a09bf3201e1fb7 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] _PACKAGE_NAMES = ('pip',) -_PIP_VERSION = "23.2.1" +_PIP_VERSION = "23.3.2" _PROJECTS = [ ("pip", _PIP_VERSION, "py3"), ] diff --git a/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl similarity index 79% rename from Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl index ba28ef02e265f0..ae78b8a6ce0737 100644 Binary files a/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index 4e76e96d2c916e..a8a50a58380375 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -4,7 +4,7 @@ __all__ = [ - 'EnumType', 'EnumMeta', + 'EnumType', 'EnumMeta', 'EnumDict', 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'ReprEnum', 'auto', 'unique', 'property', 'verify', 'member', 'nonmember', 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', @@ -313,45 +313,8 @@ def __set_name__(self, enum_class, member_name): ): # no other instances found, record this member in _member_names_ enum_class._member_names_.append(member_name) - # if necessary, get redirect in place and then add it to _member_map_ - found_descriptor = None - descriptor_type = None - class_type = None - for base in enum_class.__mro__[1:]: - attr = base.__dict__.get(member_name) - if attr is not None: - if isinstance(attr, (property, DynamicClassAttribute)): - found_descriptor = attr - class_type = base - descriptor_type = 'enum' - break - elif _is_descriptor(attr): - found_descriptor = attr - descriptor_type = descriptor_type or 'desc' - class_type = class_type or base - continue - else: - descriptor_type = 'attr' - class_type = base - if found_descriptor: - redirect = property() - redirect.member = enum_member - redirect.__set_name__(enum_class, member_name) - if descriptor_type in ('enum','desc'): - # earlier descriptor found; copy fget, fset, fdel to this one. - redirect.fget = getattr(found_descriptor, 'fget', None) - redirect._get = getattr(found_descriptor, '__get__', None) - redirect.fset = getattr(found_descriptor, 'fset', None) - redirect._set = getattr(found_descriptor, '__set__', None) - redirect.fdel = getattr(found_descriptor, 'fdel', None) - redirect._del = getattr(found_descriptor, '__delete__', None) - redirect._attr_type = descriptor_type - redirect._cls_type = class_type - setattr(enum_class, member_name, redirect) - else: - setattr(enum_class, member_name, enum_member) - # now add to _member_map_ (even aliases) - enum_class._member_map_[member_name] = enum_member + + enum_class._add_member_(member_name, enum_member) try: # This may fail if value is not hashable. We can't add the value # to the map, and by-value lookups for this value will be @@ -360,9 +323,10 @@ def __set_name__(self, enum_class, member_name): except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(member_name, []).append(value) -class _EnumDict(dict): +class EnumDict(dict): """ Track enum member order and ensure member names are not reused. @@ -371,7 +335,7 @@ class _EnumDict(dict): """ def __init__(self): super().__init__() - self._member_names = {} # use a dict to keep insertion order + self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 self._last_values = [] self._ignore = [] self._auto_called = False @@ -393,6 +357,7 @@ def __setitem__(self, key, value): '_order_', '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', + '_add_alias_', '_add_value_alias_', ): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' @@ -468,6 +433,10 @@ def __setitem__(self, key, value): self._last_values.append(value) super().__setitem__(key, value) + @property + def member_names(self): + return list(self._member_names) + def update(self, members, **more_members): try: for name in members.keys(): @@ -478,6 +447,8 @@ def update(self, members, **more_members): for name, value in more_members.items(): self[name] = value +_EnumDict = EnumDict # keep private name for backwards compatibility + class EnumType(type): """ @@ -489,7 +460,7 @@ def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members_(cls, bases) # create the namespace dict - enum_dict = _EnumDict() + enum_dict = EnumDict() enum_dict._cls_name = cls # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) @@ -552,6 +523,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_member_map_'] = {} classdict['_value2member_map_'] = {} classdict['_unhashable_values_'] = [] + classdict['_unhashable_values_map_'] = {} classdict['_member_type_'] = member_type # now set the __repr__ for the value classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) @@ -568,12 +540,16 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k try: exc = None enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) - except RuntimeError as e: - # any exceptions raised by member.__new__ will get converted to a - # RuntimeError, so get that original exception back and raise it instead - exc = e.__cause__ or e + except Exception as e: + # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError + # recreate the exception to discard + exc = type(e)(str(e)) + exc.__cause__ = e.__cause__ + exc.__context__ = e.__context__ + tb = e.__traceback__ if exc is not None: - raise exc + raise exc.with_traceback(tb) # # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__) @@ -750,7 +726,10 @@ def __contains__(cls, value): """ if isinstance(value, cls): return True - return value in cls._value2member_map_ or value in cls._unhashable_values_ + try: + return value in cls._value2member_map_ + except TypeError: + return value in cls._unhashable_values_ def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -1046,7 +1025,57 @@ def _find_new_(mcls, classdict, member_type, first_enum): else: use_args = True return __new__, save_new, use_args -EnumMeta = EnumType + + def _add_member_(cls, name, member): + # _value_ structures are not updated + if name in cls._member_map_: + if cls._member_map_[name] is not member: + raise NameError('%r is already bound: %r' % (name, cls._member_map_[name])) + return + # + # if necessary, get redirect in place and then add it to _member_map_ + found_descriptor = None + descriptor_type = None + class_type = None + for base in cls.__mro__[1:]: + attr = base.__dict__.get(name) + if attr is not None: + if isinstance(attr, (property, DynamicClassAttribute)): + found_descriptor = attr + class_type = base + descriptor_type = 'enum' + break + elif _is_descriptor(attr): + found_descriptor = attr + descriptor_type = descriptor_type or 'desc' + class_type = class_type or base + continue + else: + descriptor_type = 'attr' + class_type = base + if found_descriptor: + redirect = property() + redirect.member = member + redirect.__set_name__(cls, name) + if descriptor_type in ('enum', 'desc'): + # earlier descriptor found; copy fget, fset, fdel to this one. + redirect.fget = getattr(found_descriptor, 'fget', None) + redirect._get = getattr(found_descriptor, '__get__', None) + redirect.fset = getattr(found_descriptor, 'fset', None) + redirect._set = getattr(found_descriptor, '__set__', None) + redirect.fdel = getattr(found_descriptor, 'fdel', None) + redirect._del = getattr(found_descriptor, '__delete__', None) + redirect._attr_type = descriptor_type + redirect._cls_type = class_type + setattr(cls, name, redirect) + else: + setattr(cls, name, member) + # now add to _member_map_ (even aliases) + cls._member_map_[name] = member + # + cls._member_map_[name] = member + +EnumMeta = EnumType # keep EnumMeta name for backwards compatibility class Enum(metaclass=EnumType): @@ -1112,9 +1141,9 @@ def __new__(cls, value): pass except TypeError: # not there, now do long search -- O(n) behavior - for member in cls._member_map_.values(): - if member._value_ == value: - return member + for name, values in cls._unhashable_values_map_.items(): + if value in values: + return cls[name] # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) if not cls._member_map_: @@ -1155,6 +1184,33 @@ def __new__(cls, value): def __init__(self, *args, **kwds): pass + def _add_alias_(self, name): + self.__class__._add_member_(name, self) + + def _add_value_alias_(self, value): + cls = self.__class__ + try: + if value in cls._value2member_map_: + if cls._value2member_map_[value] is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + except TypeError: + # unhashable value, do long search + for m in cls._member_map_.values(): + if m._value_ == value: + if m is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + cls._value2member_map_.setdefault(value, self) + except TypeError: + # keep track of the value in a list so containment checks are quick + cls._unhashable_values_.append(value) + cls._unhashable_values_map_.setdefault(self.name, []).append(value) + @staticmethod def _generate_next_value_(name, start, count, last_values): """ @@ -1667,7 +1723,8 @@ def convert_class(cls): body['_member_names_'] = member_names = [] body['_member_map_'] = member_map = {} body['_value2member_map_'] = value2member_map = {} - body['_unhashable_values_'] = [] + body['_unhashable_values_'] = unhashable_values = [] + body['_unhashable_values_map_'] = {} body['_member_type_'] = member_type = etype._member_type_ body['_value_repr_'] = etype._value_repr_ if issubclass(etype, Flag): @@ -1714,14 +1771,9 @@ def convert_class(cls): for name, value in attrs.items(): if isinstance(value, auto) and auto.value is _auto_null: value = gnv(name, 1, len(member_names), gnv_last_values) - if value in value2member_map: + if value in value2member_map or value in unhashable_values: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + enum_class(value)._add_alias_(name) else: # create the member if use_args: @@ -1736,12 +1788,12 @@ def convert_class(cls): member._name_ = name member.__objclass__ = enum_class member.__init__(value) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member member._sort_order_ = len(member_names) + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) value2member_map[value] = member if _is_single_bit(value): # not a multi-bit alias, record in _member_names_ and _flag_mask_ @@ -1764,14 +1816,13 @@ def convert_class(cls): if value.value is _auto_null: value.value = gnv(name, 1, len(member_names), gnv_last_values) value = value.value - if value in value2member_map: + try: + contained = value in value2member_map + except TypeError: + contained = value in unhashable_values + if contained: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + enum_class(value)._add_alias_(name) else: # create the member if use_args: @@ -1787,14 +1838,22 @@ def convert_class(cls): member.__objclass__ = enum_class member.__init__(value) member._sort_order_ = len(member_names) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member - value2member_map[value] = member + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) member_names.append(name) gnv_last_values.append(value) + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_.setdefault(value, member) + except TypeError: + # keep track of the value in a list so containment checks are quick + enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(name, []).append(value) if '__new__' in body: enum_class.__new_member__ = enum_class.__new__ enum_class.__new__ = Enum.__new__ diff --git a/Lib/fractions.py b/Lib/fractions.py index c95db0730e5b6d..6532d5d54e3c35 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -139,6 +139,23 @@ def _round_to_figures(n, d, figures): return sign, significand, exponent +# Pattern for matching non-float-style format specifications. +_GENERAL_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" + (?: + (?P.)? + (?P[<>=^]) + )? + (?P[-+ ]?) + # Alt flag forces a slash and denominator in the output, even for + # integer-valued Fraction objects. + (?P\#)? + # We don't implement the zeropad flag since there's no single obvious way + # to interpret it. + (?P0|[1-9][0-9]*)? + (?P[,_])? +""", re.DOTALL | re.VERBOSE).fullmatch + + # Pattern for matching float-style format specifications; # supports 'e', 'E', 'f', 'F', 'g', 'G' and '%' presentation types. _FLOAT_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" @@ -414,27 +431,42 @@ def __str__(self): else: return '%s/%s' % (self._numerator, self._denominator) - def __format__(self, format_spec, /): - """Format this fraction according to the given format specification.""" - - # Backwards compatiblility with existing formatting. - if not format_spec: - return str(self) + def _format_general(self, match): + """Helper method for __format__. + Handles fill, alignment, signs, and thousands separators in the + case of no presentation type. + """ # Validate and parse the format specifier. - match = _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec) - if match is None: - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}" - ) - elif match["align"] is not None and match["zeropad"] is not None: - # Avoid the temptation to guess. - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}; " - "can't use explicit alignment when zero-padding" - ) + fill = match["fill"] or " " + align = match["align"] or ">" + pos_sign = "" if match["sign"] == "-" else match["sign"] + alternate_form = bool(match["alt"]) + minimumwidth = int(match["minimumwidth"] or "0") + thousands_sep = match["thousands_sep"] or '' + + # Determine the body and sign representation. + n, d = self._numerator, self._denominator + if d > 1 or alternate_form: + body = f"{abs(n):{thousands_sep}}/{d:{thousands_sep}}" + else: + body = f"{abs(n):{thousands_sep}}" + sign = '-' if n < 0 else pos_sign + + # Pad with fill character if necessary and return. + padding = fill * (minimumwidth - len(sign) - len(body)) + if align == ">": + return padding + sign + body + elif align == "<": + return sign + body + padding + elif align == "^": + half = len(padding) // 2 + return padding[:half] + sign + body + padding[half:] + else: # align == "=" + return sign + padding + body + + def _format_float_style(self, match): + """Helper method for __format__; handles float presentation types.""" fill = match["fill"] or " " align = match["align"] or ">" pos_sign = "" if match["sign"] == "-" else match["sign"] @@ -530,6 +562,23 @@ def __format__(self, format_spec, /): else: # align == "=" return sign + padding + body + def __format__(self, format_spec, /): + """Format this fraction according to the given format specification.""" + + if match := _GENERAL_FORMAT_SPECIFICATION_MATCHER(format_spec): + return self._format_general(match) + + if match := _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec): + # Refuse the temptation to guess if both alignment _and_ + # zero padding are specified. + if match["align"] is None or match["zeropad"] is None: + return self._format_float_style(match) + + raise ValueError( + f"Invalid format specifier {format_spec!r} " + f"for object of type {type(self).__name__!r}" + ) + def _operator_fallbacks(monomorphic_operator, fallback_operator): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. diff --git a/Lib/getpass.py b/Lib/getpass.py index 8b42c0a536b4c4..bd0097ced94c5e 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -156,7 +156,11 @@ def getuser(): First try various environment variables, then the password database. This works on Windows as long as USERNAME is set. + Any failure to find a username raises OSError. + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` + were raised. """ for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'): @@ -164,9 +168,12 @@ def getuser(): if user: return user - # If this fails, the exception will "explain" why - import pwd - return pwd.getpwuid(os.getuid())[0] + try: + import pwd + return pwd.getpwuid(os.getuid())[0] + except (ImportError, KeyError) as e: + raise OSError('No username set in the environment') from e + # Bind the name getpass to the appropriate function try: diff --git a/Lib/hmac.py b/Lib/hmac.py index 8b4f920db954ca..8b4eb2fe741e60 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -53,7 +53,7 @@ def __init__(self, key, msg=None, digestmod=''): raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) if not digestmod: - raise TypeError("Missing required parameter 'digestmod'.") + raise TypeError("Missing required argument 'digestmod'.") if _hashopenssl and isinstance(digestmod, (str, _functype)): try: diff --git a/Lib/http/client.py b/Lib/http/client.py index 7bb5d824bb6da4..5eebfccafbca59 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -665,6 +665,8 @@ def read1(self, n=-1): self._close_conn() elif self.length is not None: self.length -= len(result) + if not self.length: + self._close_conn() return result def peek(self, n=-1): @@ -689,6 +691,8 @@ def readline(self, limit=-1): self._close_conn() elif self.length is not None: self.length -= len(result) + if not self.length: + self._close_conn() return result def _read1_chunked(self, n): diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/News3.txt similarity index 99% rename from Lib/idlelib/NEWS.txt rename to Lib/idlelib/News3.txt index f258797c6e0fb3..f38cc96eceb766 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/News3.txt @@ -1,9 +1,27 @@ +What's New in IDLE 3.13.0 +(since 3.12.0) +Released on 2024-10-xx +========================= + + +gh-57795: Enter selected text into the Find box when opening +a Replace dialog. Patch by Roger Serwy and Zackery Spytz. + +gh-113269: Fix test_editor hang on macOS Catalina. +Patch by Terry Reedy. + +gh-112939: Fix processing unsaved files when quitting IDLE on macOS. +Patch by Ronald Oussoren and Christopher Chavez. + +gh-79871: Add docstrings to debugger.py. Fix two bugs in +test_debugger and expand coverage by 47%. Patch by Anthony Shaw. + + What's New in IDLE 3.12.0 (since 3.11.0) Released on 2023-10-02 ========================= - gh-104719: Remove IDLE's modification of tokenize.tabsize and test other uses of tokenize data and methods. diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 4fe64dced60aca..8b9060e57072ea 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -250,9 +250,11 @@ def closure(): class Nested_in_closure: pass ModuleBrowser(parent, file, _htest=True) + if __name__ == "__main__": if len(sys.argv) == 1: # If pass file on command line, unittest fails. from unittest import main main('idlelib.idle_test.test_browser', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(_module_browser) diff --git a/Lib/idlelib/calltip_w.py b/Lib/idlelib/calltip_w.py index 278546064adde2..9386376058c791 100644 --- a/Lib/idlelib/calltip_w.py +++ b/Lib/idlelib/calltip_w.py @@ -193,6 +193,7 @@ def calltip_hide(event): text.focus_set() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_calltip_w', verbosity=2, exit=False) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 898efeb4dd1550..92992fd9cce9cd 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -906,6 +906,7 @@ def dumpCfg(cfg): dumpCfg(idleConf.userCfg) print('\nlines = ', line, ', crc = ', crc, sep='') + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_config', verbosity=2, exit=False) diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index bb07231cd590b6..e5f67e8d4069ee 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -351,4 +351,4 @@ def cancel(self, event=None): main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(GetKeysDialog) + run(GetKeysWindow) diff --git a/Lib/idlelib/debugobj.py b/Lib/idlelib/debugobj.py index 032b686f379378..156377f8ed26ac 100644 --- a/Lib/idlelib/debugobj.py +++ b/Lib/idlelib/debugobj.py @@ -120,7 +120,7 @@ def make_objecttreeitem(labeltext, object, setfunction=None): return c(labeltext, object, setfunction) -def _object_browser(parent): # htest # +def _debug_object_browser(parent): # htest # import sys from tkinter import Toplevel top = Toplevel(parent) @@ -135,9 +135,10 @@ def _object_browser(parent): # htest # node = TreeNode(sc.canvas, None, item) node.update() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_debugobj', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_object_browser) + run(_debug_object_browser) diff --git a/Lib/idlelib/delegator.py b/Lib/idlelib/delegator.py index 55c95da8532f47..93ae8bbd43ff44 100644 --- a/Lib/idlelib/delegator.py +++ b/Lib/idlelib/delegator.py @@ -28,6 +28,7 @@ def setdelegate(self, delegate): self.resetcache() self.delegate = delegate + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_delegator', verbosity=2) diff --git a/Lib/idlelib/dynoption.py b/Lib/idlelib/dynoption.py index d5dfc3eda13f60..b8937f7106ca75 100644 --- a/Lib/idlelib/dynoption.py +++ b/Lib/idlelib/dynoption.py @@ -29,6 +29,7 @@ def SetMenu(self,valueList,value=None): if value: self.variable.set(value) + def _dyn_option_menu(parent): # htest # from tkinter import Toplevel # + StringVar, Button @@ -49,6 +50,7 @@ def update(): button = Button(top, text="Change option set", command=update) button.pack() + if __name__ == '__main__': # Only module without unittests because of intention to replace. from idlelib.idle_test.htest import run diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 69b27d0683a104..6ad383f460c7ee 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1748,6 +1748,7 @@ def _editor_window(parent): # htest # # Does not stop error, neither does following # edit.text.bind("<>", edit.close_event) + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_editor', verbosity=2, exit=False) diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py index f87781d2570fe0..e27e5d32a0ff63 100644 --- a/Lib/idlelib/filelist.py +++ b/Lib/idlelib/filelist.py @@ -124,6 +124,7 @@ def _test(): # TODO check and convert to htest if flist.inversedict: root.mainloop() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_filelist', verbosity=2) diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py index 12513594b76f8f..ef14349960bfa2 100644 --- a/Lib/idlelib/grep.py +++ b/Lib/idlelib/grep.py @@ -204,15 +204,17 @@ def _grep_dialog(parent): # htest # frame.pack() text = Text(frame, height=5) text.pack() + text.insert('1.0', 'import grep') def show_grep_dialog(): - text.tag_add(SEL, "1.0", END) + text.tag_add(SEL, "1.0", '1.end') grep(text, flist=flist) - text.tag_remove(SEL, "1.0", END) + text.tag_remove(SEL, "1.0", '1.end') button = Button(frame, text="Show GrepDialog", command=show_grep_dialog) button.pack() + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_grep', verbosity=2, exit=False) diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index cc027b9cef4f5b..3cc7e36e35555b 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -278,7 +278,8 @@ def copy_strip(): out.write(line.rstrip() + b'\n') print(f'{src} copied to {dst}') -def show_idlehelp(parent): + +def _helpwindow(parent): "Create HelpWindow; called from Idle Help event handler." filename = join(abspath(dirname(__file__)), 'help.html') if not isfile(filename): @@ -286,9 +287,10 @@ def show_idlehelp(parent): return HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version()) + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_help', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(show_idlehelp) + run(_helpwindow) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index cfa4ca781f087d..aa1c352897f9e7 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -129,11 +129,11 @@ def create_widgets(self): idle.grid(row=12, column=0, sticky=W, padx=10, pady=0) idle_buttons = Frame(frame_background, bg=self.bg) idle_buttons.grid(row=13, column=0, columnspan=3, sticky=NSEW) - self.readme = Button(idle_buttons, text='README', width=8, + self.readme = Button(idle_buttons, text='Readme', width=8, highlightbackground=self.bg, command=self.show_readme) self.readme.pack(side=LEFT, padx=10, pady=10) - self.idle_news = Button(idle_buttons, text='NEWS', width=8, + self.idle_news = Button(idle_buttons, text='News', width=8, highlightbackground=self.bg, command=self.show_idle_news) self.idle_news.pack(side=LEFT, padx=10, pady=10) @@ -167,7 +167,7 @@ def show_readme(self): def show_idle_news(self): "Handle News button event." - self.display_file_text('About - NEWS', 'NEWS.txt', 'utf-8') + self.display_file_text('About - News', 'News3.txt', 'utf-8') def display_printer_text(self, title, printer): """Create textview for built-in constants. diff --git a/Lib/idlelib/idle_test/example_stub.pyi b/Lib/idlelib/idle_test/example_stub.pyi index a9811a78d10a6e..17b58010a9d8de 100644 --- a/Lib/idlelib/idle_test/example_stub.pyi +++ b/Lib/idlelib/idle_test/example_stub.pyi @@ -1,2 +1,4 @@ +" Example to test recognition of .pyi file as Python source code. + class Example: def method(self, argument1: str, argument2: list[int]) -> None: ... diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index d297f8aa0094ee..997f85ff5a78b2 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -1,38 +1,36 @@ -'''Run human tests of Idle's window, dialog, and popup widgets. - -run(*tests) -Create a master Tk window. Within that, run each callable in tests -after finding the matching test spec in this file. If tests is empty, -run an htest for each spec dict in this file after finding the matching -callable in the module named in the spec. Close the window to skip or -end the test. - -In a tested module, let X be a global name bound to a callable (class -or function) whose .__name__ attribute is also X (the usual situation). -The first parameter of X must be 'parent'. When called, the parent -argument will be the root window. X must create a child Toplevel -window (or subclass thereof). The Toplevel may be a test widget or -dialog, in which case the callable is the corresponding class. Or the -Toplevel may contain the widget to be tested or set up a context in -which a test widget is invoked. In this latter case, the callable is a -wrapper function that sets up the Toplevel and other objects. Wrapper -function names, such as _editor_window', should start with '_'. +"""Run human tests of Idle's window, dialog, and popup widgets. + +run(*tests) Create a master Tk() htest window. Within that, run each +callable in tests after finding the matching test spec in this file. If +tests is empty, run an htest for each spec dict in this file after +finding the matching callable in the module named in the spec. Close +the master window to end testing. + +In a tested module, let X be a global name bound to a callable (class or +function) whose .__name__ attribute is also X (the usual situation). The +first parameter of X must be 'parent' or 'master'. When called, the +first argument will be the root window. X must create a child +Toplevel(parent/master) (or subclass thereof). The Toplevel may be a +test widget or dialog, in which case the callable is the corresponding +class. Or the Toplevel may contain the widget to be tested or set up a +context in which a test widget is invoked. In this latter case, the +callable is a wrapper function that sets up the Toplevel and other +objects. Wrapper function names, such as _editor_window', should start +with '_' and be lowercase. End the module with if __name__ == '__main__': - + from idlelib.idle_test.htest import run - run(X) + run(callable) # There could be multiple comma-separated callables. -To have wrapper functions and test invocation code ignored by coveragepy -reports, put '# htest #' on the def statement header line. - -def _wrapper(parent): # htest # - -Also make sure that the 'if __name__' line matches the above. Then have -make sure that .coveragerc includes the following. +To have wrapper functions ignored by coverage reports, tag the def +header like so: "def _wrapper(parent): # htest #". Use the same tag +for htest lines in widget code. Make sure that the 'if __name__' line +matches the above. Then have make sure that .coveragerc includes the +following: [report] exclude_lines = @@ -46,7 +44,7 @@ def _wrapper(parent): # htest # following template, with X.__name__ prepended to '_spec'. When all tests are run, the prefix is use to get X. -_spec = { +callable_spec = { 'file': '', 'kwds': {'title': ''}, 'msg': "" @@ -54,16 +52,16 @@ def _wrapper(parent): # htest # file (no .py): run() imports file.py. kwds: augmented with {'parent':root} and passed to X as **kwds. -title: an example kwd; some widgets need this, delete if not. +title: an example kwd; some widgets need this, delete line if not. msg: master window hints about testing the widget. -Modules and classes not being tested at the moment: -pyshell.PyShellEditorWindow -debugger.Debugger -autocomplete_w.AutoCompleteWindow -outwin.OutputWindow (indirectly being tested with grep test) -''' +TODO test these modules and classes: + autocomplete_w.AutoCompleteWindow + debugger.Debugger + outwin.OutputWindow (indirectly being tested with grep test) + pyshell.PyShellEditorWindow +""" import idlelib.pyshell # Set Windows DPI awareness before Tk(). from importlib import import_module @@ -91,15 +89,6 @@ def _wrapper(parent): # htest # "Force-open-calltip does not work here.\n" } -_module_browser_spec = { - 'file': 'browser', - 'kwds': {}, - 'msg': "Inspect names of module, class(with superclass if " - "applicable), methods and functions.\nToggle nested items.\n" - "Double clicking on items prints a traceback for an exception " - "that is ignored." - } - _color_delegator_spec = { 'file': 'colorizer', 'kwds': {}, @@ -109,16 +98,6 @@ def _wrapper(parent): # htest # "The default color scheme is in idlelib/config-highlight.def" } -CustomRun_spec = { - 'file': 'query', - 'kwds': {'title': 'Customize query.py Run', - '_htest': True}, - 'msg': "Enter with or [Run]. Print valid entry to Shell\n" - "Arguments are parsed into a list\n" - "Mode is currently restart True or False\n" - "Close dialog with valid entry, , [Cancel], [X]" - } - ConfigDialog_spec = { 'file': 'configdialog', 'kwds': {'title': 'ConfigDialogTest', @@ -135,6 +114,24 @@ def _wrapper(parent): # htest # "changes made have persisted." } +CustomRun_spec = { + 'file': 'query', + 'kwds': {'title': 'Customize query.py Run', + '_htest': True}, + 'msg': "Enter with or [OK]. Print valid entry to Shell\n" + "Arguments are parsed into a list\n" + "Mode is currently restart True or False\n" + "Close dialog with valid entry, , [Cancel], [X]" + } + +_debug_object_browser_spec = { + 'file': 'debugobj', + 'kwds': {}, + 'msg': "Double click on items up to the lowest level.\n" + "Attributes of the objects and related information " + "will be displayed side-by-side at each level." + } + # TODO Improve message _dyn_option_menu_spec = { 'file': 'dynoption', @@ -152,7 +149,7 @@ def _wrapper(parent): # htest # "Best to close editor first." } -GetKeysDialog_spec = { +GetKeysWindow_spec = { 'file': 'config_key', 'kwds': {'title': 'Test keybindings', 'action': 'find-again', @@ -173,8 +170,8 @@ def _wrapper(parent): # htest # 'msg': "Click the 'Show GrepDialog' button.\n" "Test the various 'Find-in-files' functions.\n" "The results should be displayed in a new '*Output*' window.\n" - "'Right-click'->'Go to file/line' anywhere in the search results " - "should open that file \nin a new EditorWindow." + "'Right-click'->'Go to file/line' in the search results\n " + "should open that file in a new EditorWindow." } HelpSource_spec = { @@ -190,7 +187,14 @@ def _wrapper(parent): # htest # "Any url ('www...', 'http...') is accepted.\n" "Test Browse with and without path, as cannot unittest.\n" "[Ok] or prints valid entry to shell\n" - "[Cancel] or prints None to shell" + ", [Cancel], or [X] prints None to shell" + } + +_helpwindow_spec = { + 'file': 'help', + 'kwds': {}, + 'msg': "If the help text displays, this works.\n" + "Text is selectable. Window is scrollable." } _io_binding_spec = { @@ -206,56 +210,36 @@ def _wrapper(parent): # htest # "Check that changes were saved by opening the file elsewhere." } -_linenumbers_drag_scrolling_spec = { - 'file': 'sidebar', +_multi_call_spec = { + 'file': 'multicall', 'kwds': {}, - 'msg': textwrap.dedent("""\ - 1. Click on the line numbers and drag down below the edge of the - window, moving the mouse a bit and then leaving it there for a while. - The text and line numbers should gradually scroll down, with the - selection updated continuously. - - 2. With the lines still selected, click on a line number above the - selected lines. Only the line whose number was clicked should be - selected. - - 3. Repeat step #1, dragging to above the window. The text and line - numbers should gradually scroll up, with the selection updated - continuously. - - 4. Repeat step #2, clicking a line number below the selection."""), + 'msg': "The following should trigger a print to console or IDLE Shell.\n" + "Entering and leaving the text area, key entry, ,\n" + ", , , \n" + ", and focusing elsewhere." } -_multi_call_spec = { - 'file': 'multicall', +_module_browser_spec = { + 'file': 'browser', 'kwds': {}, - 'msg': "The following actions should trigger a print to console or IDLE" - " Shell.\nEntering and leaving the text area, key entry, " - ",\n, , " - ", \n, and " - "focusing out of the window\nare sequences to be tested." + 'msg': textwrap.dedent(""" + "Inspect names of module, class(with superclass if applicable), + "methods and functions. Toggle nested items. Double clicking + "on items prints a traceback for an exception that is ignored.""") } _multistatus_bar_spec = { 'file': 'statusbar', 'kwds': {}, 'msg': "Ensure presence of multi-status bar below text area.\n" - "Click 'Update Status' to change the multi-status text" - } - -_object_browser_spec = { - 'file': 'debugobj', - 'kwds': {}, - 'msg': "Double click on items up to the lowest level.\n" - "Attributes of the objects and related information " - "will be displayed side-by-side at each level." + "Click 'Update Status' to change the status text" } -_path_browser_spec = { +PathBrowser_spec = { 'file': 'pathbrowser', - 'kwds': {}, + 'kwds': {'_htest': True}, 'msg': "Test for correct display of all paths in sys.path.\n" - "Toggle nested items up to the lowest level.\n" + "Toggle nested items out to the lowest level.\n" "Double clicking on an item prints a traceback\n" "for an exception that is ignored." } @@ -291,6 +275,15 @@ def _wrapper(parent): # htest # "Click [Close] or [X] to close the 'Replace Dialog'." } +_scrolled_list_spec = { + 'file': 'scrolledlist', + 'kwds': {}, + 'msg': "You should see a scrollable list of items\n" + "Selecting (clicking) or double clicking an item " + "prints the name to the console or Idle shell.\n" + "Right clicking an item will display a popup." + } + _search_dialog_spec = { 'file': 'search', 'kwds': {}, @@ -306,28 +299,31 @@ def _wrapper(parent): # htest # "Its only action is to close." } -_scrolled_list_spec = { - 'file': 'scrolledlist', +_sidebar_number_scrolling_spec = { + 'file': 'sidebar', 'kwds': {}, - 'msg': "You should see a scrollable list of items\n" - "Selecting (clicking) or double clicking an item " - "prints the name to the console or Idle shell.\n" - "Right clicking an item will display a popup." - } + 'msg': textwrap.dedent("""\ + 1. Click on the line numbers and drag down below the edge of the + window, moving the mouse a bit and then leaving it there for a + while. The text and line numbers should gradually scroll down, + with the selection updated continuously. -show_idlehelp_spec = { - 'file': 'help', - 'kwds': {}, - 'msg': "If the help text displays, this works.\n" - "Text is selectable. Window is scrollable." + 2. With the lines still selected, click on a line number above + or below the selected lines. Only the line whose number was + clicked should be selected. + + 3. Repeat step #1, dragging to above the window. The text and + line numbers should gradually scroll up, with the selection + updated continuously. + + 4. Repeat step #2, clicking a line number below the selection."""), } -_stack_viewer_spec = { +_stackbrowser_spec = { 'file': 'stackviewer', 'kwds': {}, 'msg': "A stacktrace for a NameError exception.\n" - "Expand 'idlelib ...' and ''.\n" - "Check that exc_value, exc_tb, and exc_type are correct.\n" + "Should have NameError and 1 traceback line." } _tooltip_spec = { @@ -370,11 +366,12 @@ def _wrapper(parent): # htest # } def run(*tests): + "Run callables in tests." root = tk.Tk() root.title('IDLE htest') root.resizable(0, 0) - # a scrollable Label like constant width text widget. + # A scrollable Label-like constant width text widget. frameLabel = tk.Frame(root, padx=10) frameLabel.pack() text = tk.Text(frameLabel, wrap='word') @@ -384,45 +381,44 @@ def run(*tests): scrollbar.pack(side='right', fill='y', expand=False) text.pack(side='left', fill='both', expand=True) - test_list = [] # List of tuples of the form (spec, callable widget) + test_list = [] # Make list of (spec, callable) tuples. if tests: for test in tests: test_spec = globals()[test.__name__ + '_spec'] test_spec['name'] = test.__name__ test_list.append((test_spec, test)) else: - for k, d in globals().items(): - if k.endswith('_spec'): - test_name = k[:-5] - test_spec = d + for key, dic in globals().items(): + if key.endswith('_spec'): + test_name = key[:-5] + test_spec = dic test_spec['name'] = test_name mod = import_module('idlelib.' + test_spec['file']) test = getattr(mod, test_name) test_list.append((test_spec, test)) + test_list.reverse() # So can pop in proper order in next_test. test_name = tk.StringVar(root) callable_object = None test_kwds = None def next_test(): - nonlocal test_name, callable_object, test_kwds if len(test_list) == 1: next_button.pack_forget() test_spec, callable_object = test_list.pop() test_kwds = test_spec['kwds'] - test_kwds['parent'] = root test_name.set('Test ' + test_spec['name']) - text.configure(state='normal') # enable text editing - text.delete('1.0','end') - text.insert("1.0",test_spec['msg']) - text.configure(state='disabled') # preserve read-only property + text['state'] = 'normal' # Enable text replacement. + text.delete('1.0', 'end') + text.insert("1.0", test_spec['msg']) + text['state'] = 'disabled' # Restore read-only property. def run_test(_=None): - widget = callable_object(**test_kwds) + widget = callable_object(root, **test_kwds) try: - print(widget.result) + print(widget.result) # Only true for query classes(?). except AttributeError: pass @@ -441,5 +437,6 @@ def close(_=None): next_test() root.mainloop() + if __name__ == '__main__': run() diff --git a/Lib/idlelib/idle_test/test_calltip.py b/Lib/idlelib/idle_test/test_calltip.py index 1ccb63b9dbd65f..15e1ff3f3cf717 100644 --- a/Lib/idlelib/idle_test/test_calltip.py +++ b/Lib/idlelib/idle_test/test_calltip.py @@ -7,6 +7,7 @@ import types import re from idlelib.idle_test.mock_tk import Text +from test.support import MISSING_C_DOCSTRINGS # Test Class TC is used in multiple get_argspec test methods @@ -50,6 +51,8 @@ class Get_argspecTest(unittest.TestCase): # but a red buildbot is better than a user crash (as has happened). # For a simple mismatch, change the expected output to the actual. + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_builtins(self): def tiptest(obj, out): @@ -143,6 +146,8 @@ def f(): pass f.__doc__ = 'a'*300 self.assertEqual(get_spec(f), f"()\n{'a'*(calltip._MAX_COLS-3) + '...'}") + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_multiline_docstring(self): # Test fewer lines than max. self.assertEqual(get_spec(range), @@ -157,6 +162,7 @@ def test_multiline_docstring(self): bytes(int) -> bytes object of size given by the parameter initialized with null bytes bytes() -> empty bytes object''') + def test_multiline_docstring_2(self): # Test more than max lines def f(): pass f.__doc__ = 'a\n' * 15 diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index 9296a6d235fbbe..0dfe2f3c58befa 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -95,7 +95,7 @@ def test_tabwidth_8(self): def insert(text, string): text.delete('1.0', 'end') text.insert('end', string) - text.update() # Force update for colorizer to finish. + text.update_idletasks() # Force update for colorizer to finish. class IndentAndNewlineTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index 8b79487b15d4cd..7e16abdb7c9f96 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -71,7 +71,7 @@ def test_file_buttons(self): """Test buttons that display files.""" dialog = self.dialog button_sources = [(self.dialog.readme, 'README.txt', 'readme'), - (self.dialog.idle_news, 'NEWS.txt', 'news'), + (self.dialog.idle_news, 'News3.txt', 'news'), (self.dialog.idle_credits, 'CREDITS.txt', 'credits')] for button, filename, name in button_sources: diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index af8159c2b33f51..464126e2df0668 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -7,7 +7,7 @@ from tkinter import filedialog from tkinter import messagebox -from tkinter.simpledialog import askstring +from tkinter.simpledialog import askstring # loadfile encoding. from idlelib.config import idleConf from idlelib.util import py_extensions @@ -180,24 +180,25 @@ def loadfile(self, filename): return True def maybesave(self): + """Return 'yes', 'no', 'cancel' as appropriate. + + Tkinter messagebox.askyesnocancel converts these tk responses + to True, False, None. Convert back, as now expected elsewhere. + """ if self.get_saved(): return "yes" - message = "Do you want to save %s before closing?" % ( - self.filename or "this untitled document") + message = ("Do you want to save " + f"{self.filename or 'this untitled document'}" + " before closing?") confirm = messagebox.askyesnocancel( title="Save On Close", message=message, default=messagebox.YES, parent=self.text) if confirm: - reply = "yes" self.save(None) - if not self.get_saved(): - reply = "cancel" - elif confirm is None: - reply = "cancel" - else: - reply = "no" + reply = "yes" if self.get_saved() else "cancel" + else: reply = "cancel" if confirm is None else "no" self.text.focus_set() return reply @@ -393,13 +394,15 @@ def updaterecentfileslist(self,filename): if self.editwin.flist: self.editwin.update_recent_files_list(filename) + def _io_binding(parent): # htest # from tkinter import Toplevel, Text - root = Toplevel(parent) - root.title("Test IOBinding") + top = Toplevel(parent) + top.title("Test IOBinding") x, y = map(int, parent.geometry().split('+')[1:]) - root.geometry("+%d+%d" % (x, y + 175)) + top.geometry("+%d+%d" % (x, y + 175)) + class MyEditWin: def __init__(self, text): self.text = text @@ -423,12 +426,13 @@ def saveas(self, event): def savecopy(self, event): self.text.event_generate("<>") - text = Text(root) + text = Text(top) text.pack() text.focus_set() editwin = MyEditWin(text) IOBinding(editwin) + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_iomenu', verbosity=2, exit=False) diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index 2ea02ec04d661a..332952f4572cbd 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -221,7 +221,7 @@ def help_dialog(event=None): # The binding above doesn't reliably work on all versions of Tk # on macOS. Adding command definition below does seem to do the # right thing for now. - root.createcommand('exit', flist.close_all_callback) + root.createcommand('::tk::mac::Quit', flist.close_all_callback) if isCarbonTk(): # for Carbon AquaTk, replace the default Tk apple menu diff --git a/Lib/idlelib/multicall.py b/Lib/idlelib/multicall.py index 0200f445cc9340..41f81813113062 100644 --- a/Lib/idlelib/multicall.py +++ b/Lib/idlelib/multicall.py @@ -421,6 +421,8 @@ def _multi_call(parent): # htest # top.geometry("+%d+%d" % (x, y + 175)) text = MultiCallCreator(tkinter.Text)(top) text.pack() + text.focus_set() + def bindseq(seq, n=[0]): def handler(event): print(seq) @@ -440,6 +442,7 @@ def handler(event): bindseq("") bindseq("") + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_mainmenu', verbosity=2, exit=False) diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index 610031e26f1dff..5ed3f35a7af655 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -182,6 +182,7 @@ def setup(self): text.tag_raise('sel') self.write = self.owin.write + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_outwin', verbosity=2, exit=False) diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index 6de242d0000bed..48a77875ba5801 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -99,13 +99,9 @@ def listmodules(self, allnames): return sorted -def _path_browser(parent): # htest # - PathBrowser(parent, _htest=True) - parent.mainloop() - if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_pathbrowser', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_path_browser) + run(PathBrowser) diff --git a/Lib/idlelib/percolator.py b/Lib/idlelib/percolator.py index 1fe34d29f54eb2..aa73427c4915c8 100644 --- a/Lib/idlelib/percolator.py +++ b/Lib/idlelib/percolator.py @@ -86,11 +86,11 @@ def delete(self, *args): print(self.name, ": delete", args) self.delegate.delete(*args) - box = tk.Toplevel(parent) - box.title("Test Percolator") + top = tk.Toplevel(parent) + top.title("Test Percolator") x, y = map(int, parent.geometry().split('+')[1:]) - box.geometry("+%d+%d" % (x, y + 175)) - text = tk.Text(box) + top.geometry("+%d+%d" % (x, y + 175)) + text = tk.Text(top) p = Percolator(text) pin = p.insertfilter pout = p.removefilter @@ -103,13 +103,15 @@ def toggle2(): (pin if var2.get() else pout)(t2) text.pack() + text.focus_set() var1 = tk.IntVar(parent) - cb1 = tk.Checkbutton(box, text="Tracer1", command=toggle1, variable=var1) + cb1 = tk.Checkbutton(top, text="Tracer1", command=toggle1, variable=var1) cb1.pack() var2 = tk.IntVar(parent) - cb2 = tk.Checkbutton(box, text="Tracer2", command=toggle2, variable=var2) + cb2 = tk.Checkbutton(top, text="Tracer2", command=toggle2, variable=var2) cb2.pack() + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_percolator', verbosity=2, exit=False) diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 00b3732a7bc4eb..1524fccd5d20f8 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -1694,6 +1694,7 @@ def main(): root.destroy() capture_warnings(False) + if __name__ == "__main__": main() diff --git a/Lib/idlelib/query.py b/Lib/idlelib/query.py index df02f2123ab02f..57230e2aaca66d 100644 --- a/Lib/idlelib/query.py +++ b/Lib/idlelib/query.py @@ -368,7 +368,7 @@ def create_extra(self): sticky='we') def cli_args_ok(self): - "Validity check and parsing for command line arguments." + "Return command line arg list or None if error." cli_string = self.entry.get().strip() try: cli_args = shlex.split(cli_string, posix=True) diff --git a/Lib/idlelib/redirector.py b/Lib/idlelib/redirector.py index 4928340e98df68..08728956abd900 100644 --- a/Lib/idlelib/redirector.py +++ b/Lib/idlelib/redirector.py @@ -164,6 +164,7 @@ def my_insert(*args): original_insert(*args) original_insert = redir.register("insert", my_insert) + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_redirector', verbosity=2, exit=False) diff --git a/Lib/idlelib/replace.py b/Lib/idlelib/replace.py index ca83173877ad1d..3716d841568d30 100644 --- a/Lib/idlelib/replace.py +++ b/Lib/idlelib/replace.py @@ -25,7 +25,8 @@ def replace(text, insert_tags=None): if not hasattr(engine, "_replacedialog"): engine._replacedialog = ReplaceDialog(root, engine) dialog = engine._replacedialog - dialog.open(text, insert_tags=insert_tags) + searchphrase = text.get("sel.first", "sel.last") + dialog.open(text, searchphrase, insert_tags=insert_tags) class ReplaceDialog(SearchDialogBase): @@ -51,27 +52,17 @@ def __init__(self, root, engine): self.replvar = StringVar(root) self.insert_tags = None - def open(self, text, insert_tags=None): + def open(self, text, searchphrase=None, *, insert_tags=None): """Make dialog visible on top of others and ready to use. - Also, highlight the currently selected text and set the - search to include the current selection (self.ok). + Also, set the search to include the current selection + (self.ok). Args: text: Text widget being searched. + searchphrase: String phrase to search. """ - SearchDialogBase.open(self, text) - try: - first = text.index("sel.first") - except TclError: - first = None - try: - last = text.index("sel.last") - except TclError: - last = None - first = first or text.index("insert") - last = last or first - self.show_hit(first, last) + SearchDialogBase.open(self, text, searchphrase) self.ok = True self.insert_tags = insert_tags @@ -120,7 +111,7 @@ def _replace_expand(self, m, repl): if self.engine.isre(): try: new = m.expand(repl) - except re.error: + except re.PatternError: self.engine.report_error(repl, 'Invalid Replace Expression') new = None else: @@ -299,6 +290,7 @@ def show_replace(): button = Button(frame, text="Replace", command=show_replace) button.pack() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_replace', verbosity=2, exit=False) diff --git a/Lib/idlelib/scrolledlist.py b/Lib/idlelib/scrolledlist.py index 71fd18ab19ec8a..4fb418db326255 100644 --- a/Lib/idlelib/scrolledlist.py +++ b/Lib/idlelib/scrolledlist.py @@ -132,6 +132,7 @@ def _scrolled_list(parent): # htest # top = Toplevel(parent) x, y = map(int, parent.geometry().split('+')[1:]) top.geometry("+%d+%d" % (x+200, y + 175)) + class MyScrolledList(ScrolledList): def fill_menu(self): self.menu.add_command(label="right click") def on_select(self, index): print("select", self.get(index)) @@ -141,9 +142,10 @@ def on_double(self, index): print("double", self.get(index)) for i in range(30): scrolled_list.append("Item %02d" % i) + if __name__ == '__main__': from unittest import main - main('idlelib.idle_test.test_scrolledlist', verbosity=2,) + main('idlelib.idle_test.test_scrolledlist', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_scrolled_list) diff --git a/Lib/idlelib/search.py b/Lib/idlelib/search.py index b35f3b59c3d2e8..935a4832257fa4 100644 --- a/Lib/idlelib/search.py +++ b/Lib/idlelib/search.py @@ -156,6 +156,7 @@ def show_find(): button = Button(frame, text="Search (selection ignored)", command=show_find) button.pack() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_search', verbosity=2, exit=False) diff --git a/Lib/idlelib/searchengine.py b/Lib/idlelib/searchengine.py index 0684142f43644a..ceb38cfaef900b 100644 --- a/Lib/idlelib/searchengine.py +++ b/Lib/idlelib/searchengine.py @@ -84,7 +84,7 @@ def getprog(self): flags = flags | re.IGNORECASE try: prog = re.compile(pat, flags) - except re.error as e: + except re.PatternError as e: self.report_error(pat, e.msg, e.pos) return None return prog diff --git a/Lib/idlelib/sidebar.py b/Lib/idlelib/sidebar.py index 166c04342907f9..ff77b568a786e0 100644 --- a/Lib/idlelib/sidebar.py +++ b/Lib/idlelib/sidebar.py @@ -513,16 +513,16 @@ def update_colors(self): self.change_callback() -def _linenumbers_drag_scrolling(parent): # htest # +def _sidebar_number_scrolling(parent): # htest # from idlelib.idle_test.test_sidebar import Dummy_editwin - toplevel = tk.Toplevel(parent) - text_frame = tk.Frame(toplevel) + top = tk.Toplevel(parent) + text_frame = tk.Frame(top) text_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True) text_frame.rowconfigure(1, weight=1) text_frame.columnconfigure(1, weight=1) - font = idleConf.GetFont(toplevel, 'main', 'EditorWindow') + font = idleConf.GetFont(top, 'main', 'EditorWindow') text = tk.Text(text_frame, width=80, height=24, wrap=tk.NONE, font=font) text.grid(row=1, column=1, sticky=tk.NSEW) @@ -540,4 +540,4 @@ def _linenumbers_drag_scrolling(parent): # htest # main('idlelib.idle_test.test_sidebar', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_linenumbers_drag_scrolling) + run(_sidebar_number_scrolling) diff --git a/Lib/idlelib/stackviewer.py b/Lib/idlelib/stackviewer.py index f8e60fd9b6d818..977c56ef15f2ae 100644 --- a/Lib/idlelib/stackviewer.py +++ b/Lib/idlelib/stackviewer.py @@ -113,7 +113,7 @@ def setfunction(value, key=key, object=self.object): return sublist -def _stack_viewer(parent): # htest # +def _stackbrowser(parent): # htest # from idlelib.pyshell import PyShellFileList top = tk.Toplevel(parent) top.title("Test StackViewer") @@ -131,4 +131,4 @@ def _stack_viewer(parent): # htest # main('idlelib.idle_test.test_stackviewer', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_stack_viewer) + run(_stackbrowser) diff --git a/Lib/idlelib/statusbar.py b/Lib/idlelib/statusbar.py index 755fafb0ac6438..8445d4cc8dfdb9 100644 --- a/Lib/idlelib/statusbar.py +++ b/Lib/idlelib/statusbar.py @@ -26,6 +26,7 @@ def _multistatus_bar(parent): # htest # x, y = map(int, parent.geometry().split('+')[1:]) top.geometry("+%d+%d" %(x, y + 175)) top.title("Test multistatus bar") + frame = Frame(top) text = Text(frame, height=5, width=40) text.pack() @@ -42,6 +43,7 @@ def change(): button.pack(side='bottom') frame.pack() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_statusbar', verbosity=2, exit=False) diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 5f30f0f6092bfa..9c2eb47b24aec9 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -492,6 +492,7 @@ def _tree_widget(parent): # htest # node = TreeNode(sc.canvas, None, item) node.expand() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_tree', verbosity=2, exit=False) diff --git a/Lib/idlelib/undo.py b/Lib/idlelib/undo.py index 5f10c0f05c1acb..f52446d5fcdcf8 100644 --- a/Lib/idlelib/undo.py +++ b/Lib/idlelib/undo.py @@ -339,25 +339,26 @@ def bump_depth(self, incr=1): def _undo_delegator(parent): # htest # from tkinter import Toplevel, Text, Button from idlelib.percolator import Percolator - undowin = Toplevel(parent) - undowin.title("Test UndoDelegator") + top = Toplevel(parent) + top.title("Test UndoDelegator") x, y = map(int, parent.geometry().split('+')[1:]) - undowin.geometry("+%d+%d" % (x, y + 175)) + top.geometry("+%d+%d" % (x, y + 175)) - text = Text(undowin, height=10) + text = Text(top, height=10) text.pack() text.focus_set() p = Percolator(text) d = UndoDelegator() p.insertfilter(d) - undo = Button(undowin, text="Undo", command=lambda:d.undo_event(None)) + undo = Button(top, text="Undo", command=lambda:d.undo_event(None)) undo.pack(side='left') - redo = Button(undowin, text="Redo", command=lambda:d.redo_event(None)) + redo = Button(top, text="Redo", command=lambda:d.redo_event(None)) redo.pack(side='left') - dump = Button(undowin, text="Dump", command=lambda:d.dump_event(None)) + dump = Button(top, text="Dump", command=lambda:d.dump_event(None)) dump.pack(side='left') + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_undo', verbosity=2, exit=False) diff --git a/Lib/idlelib/util.py b/Lib/idlelib/util.py index ede670a4db5536..a7ae74b0579004 100644 --- a/Lib/idlelib/util.py +++ b/Lib/idlelib/util.py @@ -13,8 +13,9 @@ * warning stuff (pyshell, run). """ -# .pyw is for Windows; .pyi is for stub files. -py_extensions = ('.py', '.pyw', '.pyi') # Order needed for open/save dialogs. +# .pyw is for Windows; .pyi is for typing stub files. +# The extension order is needed for iomenu open/save dialogs. +py_extensions = ('.py', '.pyw', '.pyi') if __name__ == '__main__': from unittest import main diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 4e96a916324824..d537598ac16433 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -413,6 +413,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) # Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) # Python 3.11a7 3494 (New location info table) +# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) # Python 3.12a1 3500 (Remove PRECALL opcode) # Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) # Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 5c09666b6a40d9..7b142e786e829e 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -3,7 +3,10 @@ import abc import csv import sys +import json import email +import types +import inspect import pathlib import zipfile import operator @@ -13,7 +16,6 @@ import itertools import posixpath import collections -import inspect from . import _adapters, _meta from ._collections import FreezableDefaultDict, Pair @@ -25,8 +27,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import List, Mapping, Optional, cast - +from typing import Iterable, List, Mapping, Optional, Set, Union, cast __all__ = [ 'Distribution', @@ -47,11 +48,11 @@ class PackageNotFoundError(ModuleNotFoundError): """The package was not found.""" - def __str__(self): + def __str__(self) -> str: return f"No package metadata was found for {self.name}" @property - def name(self): + def name(self) -> str: # type: ignore[override] (name,) = self.args return name @@ -117,38 +118,11 @@ def read(text, filter_=None): yield Pair(name, value) @staticmethod - def valid(line): + def valid(line: str): return line and not line.startswith('#') -class DeprecatedTuple: - """ - Provide subscript item access for backward compatibility. - - >>> recwarn = getfixture('recwarn') - >>> ep = EntryPoint(name='name', value='value', group='group') - >>> ep[:] - ('name', 'value', 'group') - >>> ep[0] - 'name' - >>> len(recwarn) - 1 - """ - - # Do not remove prior to 2023-05-01 or Python 3.13 - _warn = functools.partial( - warnings.warn, - "EntryPoint tuple interface is deprecated. Access members by name.", - DeprecationWarning, - stacklevel=2, - ) - - def __getitem__(self, item): - self._warn() - return self._key()[item] - - -class EntryPoint(DeprecatedTuple): +class EntryPoint: """An entry point as defined by Python packaging conventions. See `the packaging docs on entry points @@ -192,7 +166,7 @@ class EntryPoint(DeprecatedTuple): dist: Optional['Distribution'] = None - def __init__(self, name, value, group): + def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) def load(self): @@ -206,18 +180,21 @@ def load(self): return functools.reduce(getattr, attrs, module) @property - def module(self): + def module(self) -> str: match = self.pattern.match(self.value) + assert match is not None return match.group('module') @property - def attr(self): + def attr(self) -> str: match = self.pattern.match(self.value) + assert match is not None return match.group('attr') @property - def extras(self): + def extras(self) -> List[str]: match = self.pattern.match(self.value) + assert match is not None return re.findall(r'\w+', match.group('extras') or '') def _for(self, dist): @@ -265,7 +242,7 @@ def __repr__(self): f'group={self.group!r})' ) - def __hash__(self): + def __hash__(self) -> int: return hash(self._key()) @@ -276,7 +253,7 @@ class EntryPoints(tuple): __slots__ = () - def __getitem__(self, name): # -> EntryPoint: + def __getitem__(self, name: str) -> EntryPoint: # type: ignore[override] """ Get the EntryPoint in self matching name. """ @@ -285,6 +262,13 @@ def __getitem__(self, name): # -> EntryPoint: except StopIteration: raise KeyError(name) + def __repr__(self): + """ + Repr with classname and tuple constructor to + signal that we deviate from regular tuple behavior. + """ + return '%s(%r)' % (self.__class__.__name__, tuple(self)) + def select(self, **params): """ Select entry points from self that match the @@ -293,14 +277,14 @@ def select(self, **params): return EntryPoints(ep for ep in self if ep.matches(**params)) @property - def names(self): + def names(self) -> Set[str]: """ Return the set of all names of all entry points. """ return {ep.name for ep in self} @property - def groups(self): + def groups(self) -> Set[str]: """ Return the set of all groups of all entry points. """ @@ -321,24 +305,28 @@ def _from_text(text): class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" - def read_text(self, encoding='utf-8'): + hash: Optional["FileHash"] + size: int + dist: "Distribution" + + def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] with self.locate().open(encoding=encoding) as stream: return stream.read() - def read_binary(self): + def read_binary(self) -> bytes: with self.locate().open('rb') as stream: return stream.read() - def locate(self): + def locate(self) -> pathlib.Path: """Return a path-like object for this path""" return self.dist.locate_file(self) class FileHash: - def __init__(self, spec): + def __init__(self, spec: str) -> None: self.mode, _, self.value = spec.partition('=') - def __repr__(self): + def __repr__(self) -> str: return f'' @@ -373,14 +361,14 @@ def read_text(self, filename) -> Optional[str]: """ @abc.abstractmethod - def locate_file(self, path): + def locate_file(self, path: Union[str, os.PathLike[str]]) -> pathlib.Path: """ Given a path to a file in this distribution, return a path to it. """ @classmethod - def from_name(cls, name: str): + def from_name(cls, name: str) -> "Distribution": """Return the Distribution for the given package name. :param name: The name of the distribution package to search for. @@ -393,12 +381,12 @@ def from_name(cls, name: str): if not name: raise ValueError("A distribution name is required.") try: - return next(cls.discover(name=name)) + return next(iter(cls.discover(name=name))) except StopIteration: raise PackageNotFoundError(name) @classmethod - def discover(cls, **kwargs): + def discover(cls, **kwargs) -> Iterable["Distribution"]: """Return an iterable of Distribution objects for all packages. Pass a ``context`` or pass keyword arguments for constructing @@ -416,7 +404,7 @@ def discover(cls, **kwargs): ) @staticmethod - def at(path): + def at(path: Union[str, os.PathLike[str]]) -> "Distribution": """Return a Distribution for the indicated metadata path :param path: a string or path-like object @@ -451,7 +439,7 @@ def metadata(self) -> _meta.PackageMetadata: return _adapters.Message(email.message_from_string(text)) @property - def name(self): + def name(self) -> str: """Return the 'Name' metadata for the distribution package.""" return self.metadata['Name'] @@ -461,16 +449,16 @@ def _normalized_name(self): return Prepared.normalize(self.name) @property - def version(self): + def version(self) -> str: """Return the 'Version' metadata for the distribution package.""" return self.metadata['Version'] @property - def entry_points(self): + def entry_points(self) -> EntryPoints: return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property - def files(self): + def files(self) -> Optional[List[PackagePath]]: """Files in this distribution. :return: List of PackagePath for this distribution or None @@ -555,7 +543,7 @@ def _read_files_egginfo_sources(self): return text and map('"{}"'.format, text.splitlines()) @property - def requires(self): + def requires(self) -> Optional[List[str]]: """Generated requirements specified for this Distribution""" reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() return reqs and list(reqs) @@ -606,6 +594,16 @@ def url_req_space(req): space = url_req_space(section.value) yield section.value + space + quoted_marker(section.name) + @property + def origin(self): + return self._load_json('direct_url.json') + + def _load_json(self, filename): + return pass_none(json.loads)( + self.read_text(filename), + object_hook=lambda data: types.SimpleNamespace(**data), + ) + class DistributionFinder(MetaPathFinder): """ @@ -634,7 +632,7 @@ def __init__(self, **kwargs): vars(self).update(kwargs) @property - def path(self): + def path(self) -> List[str]: """ The sequence of directory path that a distribution finder should search. @@ -645,7 +643,7 @@ def path(self): return vars(self).get('path', sys.path) @abc.abstractmethod - def find_distributions(self, context=Context()): + def find_distributions(self, context=Context()) -> Iterable[Distribution]: """ Find distributions. @@ -774,7 +772,9 @@ def __bool__(self): class MetadataPathFinder(DistributionFinder): @classmethod - def find_distributions(cls, context=DistributionFinder.Context()): + def find_distributions( + cls, context=DistributionFinder.Context() + ) -> Iterable["PathDistribution"]: """ Find distributions. @@ -794,19 +794,19 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) - def invalidate_caches(cls): + def invalidate_caches(cls) -> None: FastPath.__new__.cache_clear() class PathDistribution(Distribution): - def __init__(self, path: SimplePath): + def __init__(self, path: SimplePath) -> None: """Construct a distribution. :param path: SimplePath indicating the metadata directory. """ self._path = path - def read_text(self, filename): + def read_text(self, filename: Union[str, os.PathLike[str]]) -> Optional[str]: with suppress( FileNotFoundError, IsADirectoryError, @@ -816,9 +816,11 @@ def read_text(self, filename): ): return self._path.joinpath(filename).read_text(encoding='utf-8') + return None + read_text.__doc__ = Distribution.read_text.__doc__ - def locate_file(self, path): + def locate_file(self, path: Union[str, os.PathLike[str]]) -> pathlib.Path: return self._path.parent / path @property @@ -851,7 +853,7 @@ def _name_from_stem(stem): return name -def distribution(distribution_name): +def distribution(distribution_name: str) -> Distribution: """Get the ``Distribution`` instance for the named package. :param distribution_name: The name of the distribution package as a string. @@ -860,7 +862,7 @@ def distribution(distribution_name): return Distribution.from_name(distribution_name) -def distributions(**kwargs): +def distributions(**kwargs) -> Iterable[Distribution]: """Get all ``Distribution`` instances in the current environment. :return: An iterable of ``Distribution`` instances. @@ -868,7 +870,7 @@ def distributions(**kwargs): return Distribution.discover(**kwargs) -def metadata(distribution_name) -> _meta.PackageMetadata: +def metadata(distribution_name: str) -> _meta.PackageMetadata: """Get the metadata for the named package. :param distribution_name: The name of the distribution package to query. @@ -877,7 +879,7 @@ def metadata(distribution_name) -> _meta.PackageMetadata: return Distribution.from_name(distribution_name).metadata -def version(distribution_name): +def version(distribution_name: str) -> str: """Get the version string for the named package. :param distribution_name: The name of the distribution package to query. @@ -911,7 +913,7 @@ def entry_points(**params) -> EntryPoints: return EntryPoints(eps).select(**params) -def files(distribution_name): +def files(distribution_name: str) -> Optional[List[PackagePath]]: """Return a list of files for the named package. :param distribution_name: The name of the distribution package to query. @@ -920,11 +922,11 @@ def files(distribution_name): return distribution(distribution_name).files -def requires(distribution_name): +def requires(distribution_name: str) -> Optional[List[str]]: """ Return a list of requirements for the named package. - :return: An iterator of requirements, suitable for + :return: An iterable of requirements, suitable for packaging.requirement.Requirement. """ return distribution(distribution_name).requires @@ -951,13 +953,42 @@ def _top_level_declared(dist): return (dist.read_text('top_level.txt') or '').split() +def _topmost(name: PackagePath) -> Optional[str]: + """ + Return the top-most parent as long as there is a parent. + """ + top, *rest = name.parts + return top if rest else None + + +def _get_toplevel_name(name: PackagePath) -> str: + """ + Infer a possibly importable module name from a name presumed on + sys.path. + + >>> _get_toplevel_name(PackagePath('foo.py')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo.pyc')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo/__init__.py')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo.pth')) + 'foo.pth' + >>> _get_toplevel_name(PackagePath('foo.dist-info')) + 'foo.dist-info' + """ + return _topmost(name) or ( + # python/typeshed#10328 + inspect.getmodulename(name) # type: ignore + or str(name) + ) + + def _top_level_inferred(dist): - opt_names = { - f.parts[0] if len(f.parts) > 1 else inspect.getmodulename(f) - for f in always_iterable(dist.files) - } + opt_names = set(map(_get_toplevel_name, always_iterable(dist.files))) - @pass_none def importable_name(name): return '.' not in name diff --git a/Lib/importlib/metadata/_adapters.py b/Lib/importlib/metadata/_adapters.py index 6aed69a30857e4..591168808953ba 100644 --- a/Lib/importlib/metadata/_adapters.py +++ b/Lib/importlib/metadata/_adapters.py @@ -53,7 +53,7 @@ def __iter__(self): def __getitem__(self, item): """ Warn users that a ``KeyError`` can be expected when a - mising key is supplied. Ref python/importlib_metadata#371. + missing key is supplied. Ref python/importlib_metadata#371. """ res = super().__getitem__(item) if res is None: diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index c9a7ef906a8a8c..f670016de7fef2 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -49,7 +49,7 @@ class SimplePath(Protocol[_T]): A minimal subset of pathlib.Path required by PathDistribution. """ - def joinpath(self) -> _T: + def joinpath(self, other: Union[str, _T]) -> _T: ... # pragma: no cover def __truediv__(self, other: Union[str, _T]) -> _T: diff --git a/Lib/importlib/metadata/diagnose.py b/Lib/importlib/metadata/diagnose.py new file mode 100644 index 00000000000000..e405471ac4d943 --- /dev/null +++ b/Lib/importlib/metadata/diagnose.py @@ -0,0 +1,21 @@ +import sys + +from . import Distribution + + +def inspect(path): + print("Inspecting", path) + dists = list(Distribution.discover(path=[path])) + if not dists: + return + print("Found", len(dists), "packages:", end=' ') + print(', '.join(dist.name for dist in dists)) + + +def run(): + for path in sys.path: + inspect(path) + + +if __name__ == '__main__': + run() diff --git a/Lib/inspect.py b/Lib/inspect.py index aaa22bef896602..f0b72662a9a0b2 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -3316,6 +3316,16 @@ def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, self) def __str__(self): + return self.format() + + def format(self, *, max_width=None): + """Create a string representation of the Signature object. + + If *max_width* integer is passed, + signature will try to fit into the *max_width*. + If signature is longer than *max_width*, + all parameters will be on separate lines. + """ result = [] render_pos_only_separator = False render_kw_only_separator = True @@ -3353,6 +3363,8 @@ def __str__(self): result.append('/') rendered = '({})'.format(', '.join(result)) + if max_width is not None and len(rendered) > max_width: + rendered = '(\n {}\n)'.format(',\n '.join(result)) if self.return_annotation is not _empty: anno = formatannotation(self.return_annotation) diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index c5d9ae2d0d5d04..d69a45d6793069 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -200,10 +200,13 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook, break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + comma_idx = end - 1 end = _w(s, end).end() nextchar = s[end:end + 1] end += 1 if nextchar != '"': + if nextchar == '}': + raise JSONDecodeError("Illegal trailing comma before end of object", s, comma_idx) raise JSONDecodeError( "Expecting property name enclosed in double quotes", s, end - 1) if object_pairs_hook is not None: @@ -240,13 +243,17 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + comma_idx = end - 1 try: if s[end] in _ws: end += 1 if s[end] in _ws: end = _w(s, end + 1).end() + nextchar = s[end:end + 1] except IndexError: pass + if nextchar == ']': + raise JSONDecodeError("Illegal trailing comma before end of array", s, comma_idx) return values, end diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 4b520e3b1e0da6..de06090942d965 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -1,4 +1,4 @@ -# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2023 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -734,7 +734,7 @@ def _configure_queue_handler(self, klass, **kwargs): lklass = kwargs['listener'] else: lklass = logging.handlers.QueueListener - listener = lklass(q, *kwargs['handlers'], respect_handler_level=rhl) + listener = lklass(q, *kwargs.get('handlers', []), respect_handler_level=rhl) handler = klass(q) handler.listener = listener return handler @@ -776,11 +776,12 @@ def configure_handler(self, config): raise ValueError('Unable to set target handler %r' % tn) from e elif issubclass(klass, logging.handlers.QueueHandler): # Another special case for handler which refers to other handlers - if 'handlers' not in config: - raise ValueError('No handlers specified for a QueueHandler') + # if 'handlers' not in config: + # raise ValueError('No handlers specified for a QueueHandler') if 'queue' in config: + from multiprocessing.queues import Queue as MPQueue qspec = config['queue'] - if not isinstance(qspec, queue.Queue): + if not isinstance(qspec, (queue.Queue, MPQueue)): if isinstance(qspec, str): q = self.resolve(qspec) if not callable(q): @@ -813,18 +814,19 @@ def configure_handler(self, config): if not callable(listener): raise TypeError('Invalid listener specifier %r' % lspec) config['listener'] = listener - hlist = [] - try: - for hn in config['handlers']: - h = self.config['handlers'][hn] - if not isinstance(h, logging.Handler): - config.update(config_copy) # restore for deferred cfg - raise TypeError('Required handler %r ' - 'is not configured yet' % hn) - hlist.append(h) - except Exception as e: - raise ValueError('Unable to set required handler %r' % hn) from e - config['handlers'] = hlist + if 'handlers' in config: + hlist = [] + try: + for hn in config['handlers']: + h = self.config['handlers'][hn] + if not isinstance(h, logging.Handler): + config.update(config_copy) # restore for deferred cfg + raise TypeError('Required handler %r ' + 'is not configured yet' % hn) + hlist.append(h) + except Exception as e: + raise ValueError('Unable to set required handler %r' % hn) from e + config['handlers'] = hlist elif issubclass(klass, logging.handlers.SMTPHandler) and\ 'mailhost' in config: config['mailhost'] = self.as_tuple(config['mailhost']) diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 36afaded705d0a..0e1d49b399d077 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -590,6 +590,8 @@ def _refresh(self): for subdir in self._toc_mtimes: path = self._paths[subdir] for entry in os.listdir(path): + if entry.startswith('.'): + continue p = os.path.join(path, entry) if os.path.isdir(p): continue @@ -1196,7 +1198,11 @@ def remove_folder(self, folder): def get_sequences(self): """Return a name-to-key-list dictionary to define each sequence.""" results = {} - with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: + try: + f = open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') + except FileNotFoundError: + return results + with f: all_keys = set(self.keys()) for line in f: try: @@ -1219,9 +1225,8 @@ def get_sequences(self): def set_sequences(self, sequences): """Set sequences using the given name-to-key-list dictionary.""" - f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') + f = open(os.path.join(self._path, '.mh_sequences'), 'w', encoding='ASCII') try: - os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) for name, keys in sequences.items(): if len(keys) == 0: continue diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index af044305709e56..49d4c7eea22411 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -101,18 +101,20 @@ def duplicate_for_child(self, handle): return reduction.duplicate(handle, self.sentinel) def wait(self, timeout=None): - if self.returncode is None: - if timeout is None: - msecs = _winapi.INFINITE - else: - msecs = max(0, int(timeout * 1000 + 0.5)) - - res = _winapi.WaitForSingleObject(int(self._handle), msecs) - if res == _winapi.WAIT_OBJECT_0: - code = _winapi.GetExitCodeProcess(self._handle) - if code == TERMINATE: - code = -signal.SIGTERM - self.returncode = code + if self.returncode is not None: + return self.returncode + + if timeout is None: + msecs = _winapi.INFINITE + else: + msecs = max(0, int(timeout * 1000 + 0.5)) + + res = _winapi.WaitForSingleObject(int(self._handle), msecs) + if res == _winapi.WAIT_OBJECT_0: + code = _winapi.GetExitCodeProcess(self._handle) + if code == TERMINATE: + code = -signal.SIGTERM + self.returncode = code return self.returncode @@ -120,18 +122,22 @@ def poll(self): return self.wait(timeout=0) def terminate(self): - if self.returncode is None: - try: - _winapi.TerminateProcess(int(self._handle), TERMINATE) - except PermissionError: - # ERROR_ACCESS_DENIED (winerror 5) is received when the - # process already died. - code = _winapi.GetExitCodeProcess(int(self._handle)) - if code == _winapi.STILL_ACTIVE: - raise - self.returncode = code - else: - self.returncode = -signal.SIGTERM + if self.returncode is not None: + return + + try: + _winapi.TerminateProcess(int(self._handle), TERMINATE) + except PermissionError: + # ERROR_ACCESS_DENIED (winerror 5) is received when the + # process already died. + code = _winapi.GetExitCodeProcess(int(self._handle)) + if code == _winapi.STILL_ACTIVE: + raise + + # gh-113009: Don't set self.returncode. Even if GetExitCodeProcess() + # returns an exit code different than STILL_ACTIVE, the process can + # still be running. Only set self.returncode once WaitForSingleObject() + # returns WAIT_OBJECT_0 in wait(). kill = terminate diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 9a1e5aa17b87a2..67e70fdc27cf31 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -71,8 +71,9 @@ class SharedMemory: _flags = os.O_RDWR _mode = 0o600 _prepend_leading_slash = True if _USE_POSIX else False + _track = True - def __init__(self, name=None, create=False, size=0): + def __init__(self, name=None, create=False, size=0, *, track=True): if not size >= 0: raise ValueError("'size' must be a positive integer") if create: @@ -82,6 +83,7 @@ def __init__(self, name=None, create=False, size=0): if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") + self._track = track if _USE_POSIX: # POSIX Shared Memory @@ -116,8 +118,8 @@ def __init__(self, name=None, create=False, size=0): except OSError: self.unlink() raise - - resource_tracker.register(self._name, "shared_memory") + if self._track: + resource_tracker.register(self._name, "shared_memory") else: @@ -236,12 +238,20 @@ def close(self): def unlink(self): """Requests that the underlying shared memory block be destroyed. - In order to ensure proper cleanup of resources, unlink should be - called once (and only once) across all processes which have access - to the shared memory block.""" + Unlink should be called once (and only once) across all handles + which have access to the shared memory block, even if these + handles belong to different processes. Closing and unlinking may + happen in any order, but trying to access data inside a shared + memory block after unlinking may result in memory errors, + depending on platform. + + This method has no effect on Windows, where the only way to + delete a shared memory block is to close all handles.""" + if _USE_POSIX and self._name: _posixshmem.shm_unlink(self._name) - resource_tracker.unregister(self._name, "shared_memory") + if self._track: + resource_tracker.unregister(self._name, "shared_memory") _encoding = "utf8" diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 28c77df1c32ea8..32871850ddec8b 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -43,19 +43,19 @@ def sub_debug(msg, *args): if _logger: - _logger.log(SUBDEBUG, msg, *args) + _logger.log(SUBDEBUG, msg, *args, stacklevel=2) def debug(msg, *args): if _logger: - _logger.log(DEBUG, msg, *args) + _logger.log(DEBUG, msg, *args, stacklevel=2) def info(msg, *args): if _logger: - _logger.log(INFO, msg, *args) + _logger.log(INFO, msg, *args, stacklevel=2) def sub_warning(msg, *args): if _logger: - _logger.log(SUBWARNING, msg, *args) + _logger.log(SUBWARNING, msg, *args, stacklevel=2) def get_logger(): ''' diff --git a/Lib/os.py b/Lib/os.py index a17946750ea7e7..7f38e14e7bdd96 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -131,6 +131,7 @@ def _add(str, fn): _set = set() _add("HAVE_FCHDIR", "chdir") _add("HAVE_FCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") _add("HAVE_FCHOWN", "chown") _add("HAVE_FDOPENDIR", "listdir") _add("HAVE_FDOPENDIR", "scandir") @@ -171,6 +172,7 @@ def _add(str, fn): _add("HAVE_FSTATAT", "stat") _add("HAVE_LCHFLAGS", "chflags") _add("HAVE_LCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") if _exists("lchown"): # mac os x10.3 _add("HAVE_LCHOWN", "chown") _add("HAVE_LINKAT", "link") diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py new file mode 100644 index 00000000000000..2b4193c400a099 --- /dev/null +++ b/Lib/pathlib/__init__.py @@ -0,0 +1,535 @@ +"""Object-oriented filesystem paths. + +This module provides classes to represent abstract paths and concrete +paths with operations that have semantics appropriate for different +operating systems. +""" + +import io +import ntpath +import os +import posixpath + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from . import _abc + + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +UnsupportedOperation = _abc.UnsupportedOperation + + +class PurePath(_abc.PurePathBase): + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + pathmod = os.path + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.pathmod is ntpath and self.pathmod is posixpath: + # GH-103631: Convert separators for backwards compatibility. + paths.extend(path.replace('\\', '/') for path in arg._raw_paths) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + # Avoid calling super().__init__, as an optimisation + self._raw_paths = paths + self._resolving = False + + def __reduce__(self): + # Using the parts tuple helps share interned path parts + # when pickling related paths. + return (self.__class__, self.parts) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if _abc._is_case_sensitive(self.pathmod): + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + pathmod = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + pathmod = ntpath + __slots__ = () + + +class Path(_abc.PathBase, PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + as_uri = PurePath.as_uri + + @classmethod + def _unsupported(cls, method_name): + msg = f"{cls.__name__}.{method_name}() is unsupported on this system" + raise UnsupportedOperation(msg) + + def __init__(self, *args, **kwargs): + if kwargs: + import warnings + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return _abc.PathBase.read_text(self, encoding, errors, newline) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return _abc.PathBase.write_text(self, data, encoding, errors, newline) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return (self._make_child_relpath(name) for name in os.listdir(self)) + + def _scandir(self): + return os.scandir(self) + + def _make_child_entry(self, entry): + # Transform an entry yielded from _scandir() into a path object. + return self._make_child_relpath(entry.name) + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + result = self.with_segments(cwd) + result._str = cwd + return result + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.pathmod.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + return self.with_segments(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib.py b/Lib/pathlib/_abc.py similarity index 68% rename from Lib/pathlib.py rename to Lib/pathlib/_abc.py index 81f75cd47ed087..f75b20a1d5f1e5 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib/_abc.py @@ -1,14 +1,5 @@ -"""Object-oriented filesystem paths. - -This module provides classes to represent abstract paths and concrete -paths with operations that have semantics appropriate for different -operating systems. -""" - import functools -import io import ntpath -import os import posixpath import sys import warnings @@ -17,27 +8,11 @@ from itertools import chain from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - - -__all__ = [ - "UnsupportedOperation", - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - # # Internals # -# Maximum number of symlinks to follow in _PathBase.resolve() +# Maximum number of symlinks to follow in PathBase.resolve() _MAX_SYMLINKS = 40 # Reference for Windows paths can be found at @@ -112,9 +87,8 @@ def _select_children(parent_paths, dir_only, follow_symlinks, match): continue except OSError: continue - name = entry.name - if match(name): - yield parent_path._make_child_relpath(name) + if match(entry.name): + yield parent_path._make_child_entry(entry) def _select_recursive(parent_paths, dir_only, follow_symlinks): @@ -137,12 +111,12 @@ def _select_recursive(parent_paths, dir_only, follow_symlinks): for entry in entries: try: if entry.is_dir(follow_symlinks=follow_symlinks): - paths.append(path._make_child_relpath(entry.name)) + paths.append(path._make_child_entry(entry)) continue except OSError: pass if not dir_only: - yield path._make_child_relpath(entry.name) + yield path._make_child_entry(entry) def _select_unique(paths): @@ -158,10 +132,6 @@ def _select_unique(paths): yielded.clear() -# -# Public API -# - class UnsupportedOperation(NotImplementedError): """An exception that is raised when an unsupported operation is called on a path object. @@ -198,14 +168,13 @@ def __repr__(self): return "<{}.parents>".format(type(self._path).__name__) -class PurePath: - """Base class for manipulating paths without I/O. +class PurePathBase: + """Base class for pure path objects. - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. + This class *does not* provide several magic methods that are defined in + its subclass PurePath. They are: __fspath__, __bytes__, __reduce__, + __hash__, __eq__, __lt__, __le__, __gt__, __ge__. Its initializer and path + joining methods accept only strings, not os.PathLike objects more broadly. """ __slots__ = ( @@ -227,28 +196,16 @@ class PurePath: # for the first time. It's used to implement `_str_normcase` '_str', - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - # The '_resolving' slot stores a boolean indicating whether the path - # is being processed by `_PathBase.resolve()`. This prevents duplicate + # is being processed by `PathBase.resolve()`. This prevents duplicate # work from occurring when `resolve()` calls `stat()` or `readlink()`. '_resolving', ) - pathmod = os.path + pathmod = posixpath + + def __init__(self, *paths): + self._raw_paths = paths + self._resolving = False def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. @@ -322,9 +279,6 @@ def as_posix(self): slashes.""" return str(self).replace(self.pathmod.sep, '/') - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - @property def drive(self): """The drive prefix (letter or UNC path), if any.""" @@ -440,11 +394,10 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): if _deprecated: msg = ("support for supplying more than one positional argument " "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, - remove=(3, 14)) + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): + elif not isinstance(other, PurePathBase): other = self.with_segments(other) for step, path in enumerate(chain([other], other.parents)): if path == self or path in self.parents: @@ -464,11 +417,10 @@ def is_relative_to(self, other, /, *_deprecated): if _deprecated: msg = ("support for supplying more than one argument to " "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", - msg, remove=(3, 14)) + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): + elif not isinstance(other, PurePathBase): other = self.with_segments(other) return other == self or other in self.parents @@ -487,7 +439,7 @@ def joinpath(self, *pathsegments): paths) or a totally different path (if one of the arguments is anchored). """ - return self.with_segments(self, *pathsegments) + return self.with_segments(*self._raw_paths, *pathsegments) def __truediv__(self, key): try: @@ -497,7 +449,7 @@ def __truediv__(self, key): def __rtruediv__(self, key): try: - return self.with_segments(key, self) + return self.with_segments(key, *self._raw_paths) except TypeError: return NotImplemented @@ -555,7 +507,7 @@ def match(self, path_pattern, *, case_sensitive=None): """ Return True if this path matches the given pattern. """ - if not isinstance(path_pattern, PurePath): + if not isinstance(path_pattern, PurePathBase): path_pattern = self.with_segments(path_pattern) if case_sensitive is None: case_sensitive = _is_case_sensitive(self.pathmod) @@ -570,156 +522,9 @@ def match(self, path_pattern, *, case_sensitive=None): match = _compile_pattern(pattern_str, sep, case_sensitive) return match(str(self)) is not None - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def __init__(self, *args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - if arg.pathmod is ntpath and self.pathmod is posixpath: - # GH-103631: Convert separators for backwards compatibility. - paths.extend(path.replace('\\', '/') for path in arg._raw_paths) - else: - paths.extend(arg._raw_paths) - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - paths.append(path) - self._raw_paths = paths - self._resolving = False - - def __reduce__(self): - # Using the parts tuple helps share interned path parts - # when pickling related paths. - return (self.__class__, self.parts) - - def __fspath__(self): - return str(self) - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - if _is_case_sensitive(self.pathmod): - self._str_normcase_cached = str(self) - else: - self._str_normcase_cached = str(self).lower() - return self._str_normcase_cached - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) - return self._parts_normcase_cached - - def __lt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - def as_uri(self): - """Return the path as a URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - from urllib.parse import quote_from_bytes - return prefix + quote_from_bytes(os.fsencode(path)) - - -# Subclassing os.PathLike makes isinstance() checks slower, -# which in turn makes Path construction slower. Register instead! -os.PathLike.register(PurePath) -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = ntpath - __slots__ = () - - -# Filesystem-accessing classes - - -class _PathBase(PurePath): +class PathBase(PurePathBase): """Base class for concrete path objects. This class provides dummy implementations for many methods that derived @@ -733,14 +538,10 @@ class _PathBase(PurePath): such as paths in archive files or on remote storage systems. """ __slots__ = () - __bytes__ = None - __fspath__ = None # virtual paths have no local file system representation @classmethod def _unsupported(cls, method_name): msg = f"{cls.__name__}.{method_name}() is unsupported" - if issubclass(cls, Path): - msg += " on this system" raise UnsupportedOperation(msg) def stat(self, *, follow_symlinks=True): @@ -950,7 +751,6 @@ def read_text(self, encoding=None, errors=None, newline=None): """ Open the file in text mode, read it, and close the file. """ - encoding = io.text_encoding(encoding) with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: return f.read() @@ -970,7 +770,6 @@ def write_text(self, data, encoding=None, errors=None, newline=None): if not isinstance(data, str): raise TypeError('data must be str, not %s' % data.__class__.__name__) - encoding = io.text_encoding(encoding) with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) @@ -988,6 +787,10 @@ def _scandir(self): from contextlib import nullcontext return nullcontext(self.iterdir()) + def _make_child_entry(self, entry): + # Transform an entry yielded from _scandir() into a path object. + return entry + def _make_child_relpath(self, name): path_str = str(self) tail = self._tail @@ -1319,13 +1122,13 @@ def rmdir(self): """ self._unsupported("rmdir") - def owner(self): + def owner(self, *, follow_symlinks=True): """ Return the login name of the file owner. """ self._unsupported("owner") - def group(self): + def group(self, *, follow_symlinks=True): """ Return the group name of the file gid. """ @@ -1339,291 +1142,3 @@ def from_uri(cls, uri): def as_uri(self): """Return the path as a URI.""" self._unsupported("as_uri") - - -class Path(_PathBase): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - __bytes__ = PurePath.__bytes__ - __fspath__ = PurePath.__fspath__ - as_uri = PurePath.as_uri - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def is_mount(self): - """ - Check if this path is a mount point - """ - return os.path.ismount(self) - - def is_junction(self): - """ - Whether this path is a junction. - """ - return os.path.isjunction(self) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - return (self._make_child_relpath(name) for name in os.listdir(self)) - - def _scandir(self): - return os.scandir(self) - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - elif self.drive: - # There is a CWD on each drive-letter drive. - cwd = os.path.abspath(self.drive) - else: - cwd = os.getcwd() - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - if not self.root and not self._tail: - result = self.with_segments(cwd) - result._str = cwd - return result - return self.with_segments(cwd, self) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - return self.with_segments(os.path.realpath(self, strict=strict)) - - if pwd: - def owner(self): - """ - Return the login name of the file owner. - """ - return pwd.getpwuid(self.stat().st_uid).pw_name - - if grp: - def group(self): - """ - Return the group name of the file gid. - """ - return grp.getgrgid(self.stat().st_gid).gr_name - - if hasattr(os, "readlink"): - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - if hasattr(os, "symlink"): - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - os.symlink(target, self, target_is_directory) - - if hasattr(os, "link"): - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = os.path.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, tail + self._tail[1:]) - - return self - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - if not uri.startswith('file:'): - raise ValueError(f"URI does not start with 'file:': {uri!r}") - path = uri[5:] - if path[:3] == '///': - # Remove empty authority - path = path[2:] - elif path[:12] == '//localhost/': - # Remove 'localhost' authority - path = path[11:] - if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): - # Remove slash before DOS device/UNC path - path = path[1:] - if path[1:2] == '|': - # Replace bar with colon in DOS drive - path = path[:1] + ':' + path[2:] - from urllib.parse import unquote_to_bytes - path = cls(os.fsdecode(unquote_to_bytes(path))) - if not path.is_absolute(): - raise ValueError(f"URI is not absolute: {uri!r}") - return path - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pdb.py b/Lib/pdb.py index ed78d749a47fa8..83b7fefec63636 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -142,8 +142,10 @@ def check(self): print('Error:', self.orig, 'is a directory') sys.exit(1) - # Replace pdb's dir with script's dir in front of module search path. - sys.path[0] = os.path.dirname(self) + # If safe_path(-P) is not set, sys.path[0] is the directory + # of pdb, and we should replace it with the directory of the script + if not sys.flags.safe_path: + sys.path[0] = os.path.dirname(self) @property def filename(self): @@ -205,6 +207,15 @@ def namespace(self): ) +class _PdbInteractiveConsole(code.InteractiveConsole): + def __init__(self, ns, message): + self._message = message + super().__init__(locals=ns, local_exit=True) + + def write(self, data): + self._message(data, end='') + + # Interaction prompt line will separate file and call info from code # text using value of line_prefix string. A newline and arrow may # be to your liking. You can set it once pdb is imported using the @@ -670,8 +681,8 @@ def handle_command_def(self, line): # interface abstraction functions - def message(self, msg): - print(msg, file=self.stdout) + def message(self, msg, end='\n'): + print(msg, end=end, file=self.stdout) def error(self, msg): print('***', msg, file=self.stdout) @@ -1784,7 +1795,9 @@ def do_interact(self, arg): contains all the (global and local) names found in the current scope. """ ns = {**self.curframe.f_globals, **self.curframe_locals} - code.interact("*interactive*", local=ns, local_exit=True) + console = _PdbInteractiveConsole(ns, message=self.message) + console.interact(banner="*pdb interact start*", + exitmsg="*exit from pdb interact command*") def do_alias(self, arg): """alias [name [command]] diff --git a/Lib/pickle.py b/Lib/pickle.py index 4f5ad5b71e8899..988c0887341310 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -857,13 +857,13 @@ def save_str(self, obj): else: self.write(BINUNICODE + pack("\d\d\d\d)(?:-(?P\d\d)(?:-(?P\d\d)(?:T(?P\d\d)(?::(?P\d\d)(?::(?P\d\d))?)?)?)?)?Z", re.ASCII) -def _date_from_string(s): +def _date_from_string(s, aware_datetime): order = ('year', 'month', 'day', 'hour', 'minute', 'second') gd = _dateParser.match(s).groupdict() lst = [] @@ -149,10 +149,14 @@ def _date_from_string(s): if val is None: break lst.append(int(val)) + if aware_datetime: + return datetime.datetime(*lst, tzinfo=datetime.UTC) return datetime.datetime(*lst) -def _date_to_string(d): +def _date_to_string(d, aware_datetime): + if aware_datetime: + d = d.astimezone(datetime.UTC) return '%04d-%02d-%02dT%02d:%02d:%02dZ' % ( d.year, d.month, d.day, d.hour, d.minute, d.second @@ -171,11 +175,12 @@ def _escape(text): return text class _PlistParser: - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self.stack = [] self.current_key = None self.root = None self._dict_type = dict_type + self._aware_datetime = aware_datetime def parse(self, fileobj): self.parser = ParserCreate() @@ -277,7 +282,8 @@ def end_data(self): self.add_object(_decode_base64(self.get_data())) def end_date(self): - self.add_object(_date_from_string(self.get_data())) + self.add_object(_date_from_string(self.get_data(), + aware_datetime=self._aware_datetime)) class _DumbXMLWriter: @@ -321,13 +327,14 @@ def writeln(self, line): class _PlistWriter(_DumbXMLWriter): def __init__( self, file, indent_level=0, indent=b"\t", writeHeader=1, - sort_keys=True, skipkeys=False): + sort_keys=True, skipkeys=False, aware_datetime=False): if writeHeader: file.write(PLISTHEADER) _DumbXMLWriter.__init__(self, file, indent_level, indent) self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): self.writeln("") @@ -360,7 +367,8 @@ def write_value(self, value): self.write_bytes(value) elif isinstance(value, datetime.datetime): - self.simple_element("date", _date_to_string(value)) + self.simple_element("date", + _date_to_string(value, self._aware_datetime)) elif isinstance(value, (tuple, list)): self.write_array(value) @@ -461,8 +469,9 @@ class _BinaryPlistParser: see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c """ - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self._dict_type = dict_type + self._aware_datime = aware_datetime def parse(self, fp): try: @@ -556,8 +565,11 @@ def _read_object(self, ref): f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. - result = (datetime.datetime(2001, 1, 1) + - datetime.timedelta(seconds=f)) + if self._aware_datime: + epoch = datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + else: + epoch = datetime.datetime(2001, 1, 1) + result = epoch + datetime.timedelta(seconds=f) elif tokenH == 0x40: # data s = self._get_size(tokenL) @@ -629,10 +641,11 @@ def _count_to_size(count): _scalars = (str, int, float, datetime.datetime, bytes) class _BinaryPlistWriter (object): - def __init__(self, fp, sort_keys, skipkeys): + def __init__(self, fp, sort_keys, skipkeys, aware_datetime=False): self._fp = fp self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): @@ -778,7 +791,12 @@ def _write_object(self, value): self._fp.write(struct.pack('>Bd', 0x23, value)) elif isinstance(value, datetime.datetime): - f = (value - datetime.datetime(2001, 1, 1)).total_seconds() + if self._aware_datetime: + dt = value.astimezone(datetime.UTC) + offset = dt - datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + f = offset.total_seconds() + else: + f = (value - datetime.datetime(2001, 1, 1)).total_seconds() self._fp.write(struct.pack('>Bd', 0x33, f)) elif isinstance(value, (bytes, bytearray)): @@ -862,7 +880,7 @@ def _is_fmt_binary(header): } -def load(fp, *, fmt=None, dict_type=dict): +def load(fp, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file. 'fp' should be a readable and binary file object. Return the unpacked root object (which usually is a dictionary). """ @@ -880,32 +898,36 @@ def load(fp, *, fmt=None, dict_type=dict): else: P = _FORMATS[fmt]['parser'] - p = P(dict_type=dict_type) + p = P(dict_type=dict_type, aware_datetime=aware_datetime) return p.parse(fp) -def loads(value, *, fmt=None, dict_type=dict): +def loads(value, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file from a bytes object. Return the unpacked root object (which usually is a dictionary). """ fp = BytesIO(value) - return load(fp, fmt=fmt, dict_type=dict_type) + return load(fp, fmt=fmt, dict_type=dict_type, aware_datetime=aware_datetime) -def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False): +def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, + aware_datetime=False): """Write 'value' to a .plist file. 'fp' should be a writable, binary file object. """ if fmt not in _FORMATS: raise ValueError("Unsupported format: %r"%(fmt,)) - writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys) + writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys, + aware_datetime=aware_datetime) writer.write(value) -def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True): +def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True, + aware_datetime=False): """Return a bytes object with the contents for a .plist file. """ fp = BytesIO() - dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys) + dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys, + aware_datetime=aware_datetime) return fp.getvalue() diff --git a/Lib/pprint.py b/Lib/pprint.py index 34ed12637e2288..9314701db340c7 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -128,6 +128,9 @@ def __init__(self, indent=1, width=80, depth=None, stream=None, *, sort_dicts If true, dict keys are sorted. + underscore_numbers + If true, digit groups are separated with underscores. + """ indent = int(indent) width = int(width) diff --git a/Lib/pstats.py b/Lib/pstats.py index 51bcca84188740..2f054bb4011e7f 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -329,7 +329,7 @@ def eval_print_amount(self, sel, list, msg): if isinstance(sel, str): try: rex = re.compile(sel) - except re.error: + except re.PatternError: msg += " \n" % sel return new_list, msg new_list = [] diff --git a/Lib/pydoc.py b/Lib/pydoc.py index be41592cc64bad..96aa1dfc1aacf6 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -201,7 +201,10 @@ def _getargspec(object): try: signature = inspect.signature(object) if signature: - return str(signature) + name = getattr(object, '__name__', '') + # function are always single-line and should not be formatted + max_width = (80 - len(name)) if name != '' else None + return signature.format(max_width=max_width) except (ValueError, TypeError): argspec = getattr(object, '__text_signature__', None) if argspec: @@ -342,6 +345,8 @@ def sort_attributes(attrs, object): def ispackage(path): """Guess whether a path refers to a package directory.""" + warnings.warn('The pydoc.ispackage() function is deprecated', + DeprecationWarning, stacklevel=2) if os.path.isdir(path): for ext in ('.py', '.pyc'): if os.path.isfile(os.path.join(path, '__init__' + ext)): diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index 428d1b0d5fbd87..7e8abbf6ffe155 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -117,7 +117,8 @@ U UNICODE For compatibility only. Ignored for string patterns (it is the default), and forbidden for bytes patterns. -This module also defines an exception 'error'. +This module also defines exception 'PatternError', aliased to 'error' for +backward compatibility. """ @@ -133,7 +134,7 @@ "findall", "finditer", "compile", "purge", "escape", "error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U", "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", - "UNICODE", "NOFLAG", "RegexFlag", + "UNICODE", "NOFLAG", "RegexFlag", "PatternError" ] __version__ = "2.2.1" @@ -155,7 +156,7 @@ class RegexFlag: _numeric_repr_ = hex # sre exception -error = _compiler.error +PatternError = error = _compiler.PatternError # -------------------------------------------------------------------- # public interface diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index f87712d6d6f9f8..7b888f877eb3dc 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -150,7 +150,7 @@ def _compile(code, pattern, flags): if lo > MAXCODE: raise error("looks too much behind") if lo != hi: - raise error("look-behind requires fixed-width pattern") + raise PatternError("look-behind requires fixed-width pattern") emit(lo) # look behind _compile(code, av[1], flags) emit(SUCCESS) @@ -209,7 +209,7 @@ def _compile(code, pattern, flags): else: code[skipyes] = _len(code) - skipyes + 1 else: - raise error("internal: unsupported operand type %r" % (op,)) + raise PatternError(f"internal: unsupported operand type {op!r}") def _compile_charset(charset, flags, code): # compile charset subprogram @@ -235,7 +235,7 @@ def _compile_charset(charset, flags, code): else: emit(av) else: - raise error("internal: unsupported set operator %r" % (op,)) + raise PatternError(f"internal: unsupported set operator {op!r}") emit(FAILURE) def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index d8e483ac4f23b4..9c3c294ba448b4 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -20,7 +20,7 @@ # SRE standard exception (access as sre.error) # should this really be here? -class error(Exception): +class PatternError(Exception): """Exception raised for invalid regular expressions. Attributes: @@ -53,6 +53,9 @@ def __init__(self, msg, pattern=None, pos=None): super().__init__(msg) +# Backward compatibility after renaming in 3.13 +error = PatternError + class _NamedIntConstant(int): def __new__(cls, value, name): self = super(_NamedIntConstant, cls).__new__(cls, value) diff --git a/Lib/shutil.py b/Lib/shutil.py index 0fed0117a63234..acc9419be4dfca 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -306,7 +306,12 @@ def copymode(src, dst, *, follow_symlinks=True): else: return else: - stat_func, chmod_func = _stat, os.chmod + stat_func = _stat + if os.name == 'nt' and os.path.islink(dst): + def chmod_func(*args): + os.chmod(*args, follow_symlinks=True) + else: + chmod_func = os.chmod st = stat_func(src) chmod_func(dst, stat.S_IMODE(st.st_mode)) @@ -590,23 +595,21 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, dirs_exist_ok=dirs_exist_ok) if hasattr(os.stat_result, 'st_file_attributes'): - def _rmtree_islink(path): - try: - st = os.lstat(path) - return (stat.S_ISLNK(st.st_mode) or - (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT - and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) - except OSError: - return False + def _rmtree_islink(st): + return (stat.S_ISLNK(st.st_mode) or + (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT + and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) else: - def _rmtree_islink(path): - return os.path.islink(path) + def _rmtree_islink(st): + return stat.S_ISLNK(st.st_mode) # version vulnerable to race conditions def _rmtree_unsafe(path, onexc): try: with os.scandir(path) as scandir_it: entries = list(scandir_it) + except FileNotFoundError: + return except OSError as err: onexc(os.scandir, path, err) entries = [] @@ -614,6 +617,8 @@ def _rmtree_unsafe(path, onexc): fullname = entry.path try: is_dir = entry.is_dir(follow_symlinks=False) + except FileNotFoundError: + continue except OSError: is_dir = False @@ -624,6 +629,8 @@ def _rmtree_unsafe(path, onexc): # a directory with a symlink after the call to # os.scandir or entry.is_dir above. raise OSError("Cannot call rmtree on a symbolic link") + except FileNotFoundError: + continue except OSError as err: onexc(os.path.islink, fullname, err) continue @@ -631,10 +638,14 @@ def _rmtree_unsafe(path, onexc): else: try: os.unlink(fullname) + except FileNotFoundError: + continue except OSError as err: onexc(os.unlink, fullname, err) try: os.rmdir(path) + except FileNotFoundError: + pass except OSError as err: onexc(os.rmdir, path, err) @@ -643,6 +654,8 @@ def _rmtree_safe_fd(topfd, path, onexc): try: with os.scandir(topfd) as scandir_it: entries = list(scandir_it) + except FileNotFoundError: + return except OSError as err: err.filename = path onexc(os.scandir, path, err) @@ -651,6 +664,8 @@ def _rmtree_safe_fd(topfd, path, onexc): fullname = os.path.join(path, entry.name) try: is_dir = entry.is_dir(follow_symlinks=False) + except FileNotFoundError: + continue except OSError: is_dir = False else: @@ -658,6 +673,8 @@ def _rmtree_safe_fd(topfd, path, onexc): try: orig_st = entry.stat(follow_symlinks=False) is_dir = stat.S_ISDIR(orig_st.st_mode) + except FileNotFoundError: + continue except OSError as err: onexc(os.lstat, fullname, err) continue @@ -665,6 +682,8 @@ def _rmtree_safe_fd(topfd, path, onexc): try: dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) dirfd_closed = False + except FileNotFoundError: + continue except OSError as err: onexc(os.open, fullname, err) else: @@ -673,8 +692,15 @@ def _rmtree_safe_fd(topfd, path, onexc): _rmtree_safe_fd(dirfd, fullname, onexc) try: os.close(dirfd) + except OSError as err: + # close() should not be retried after an error. dirfd_closed = True + onexc(os.close, fullname, err) + dirfd_closed = True + try: os.rmdir(entry.name, dir_fd=topfd) + except FileNotFoundError: + continue except OSError as err: onexc(os.rmdir, fullname, err) else: @@ -688,10 +714,15 @@ def _rmtree_safe_fd(topfd, path, onexc): onexc(os.path.islink, fullname, err) finally: if not dirfd_closed: - os.close(dirfd) + try: + os.close(dirfd) + except OSError as err: + onexc(os.close, fullname, err) else: try: os.unlink(entry.name, dir_fd=topfd) + except FileNotFoundError: + continue except OSError as err: onexc(os.unlink, fullname, err) @@ -721,11 +752,6 @@ def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): If both onerror and onexc are set, onerror is ignored and onexc is used. """ - if onerror is not None: - import warnings - warnings.warn("onerror argument is deprecated, use onexc instead", - DeprecationWarning, stacklevel=2) - sys.audit("shutil.rmtree", path, dir_fd) if ignore_errors: def onexc(*args): @@ -755,13 +781,13 @@ def onexc(*args): # lstat()/open()/fstat() trick. try: orig_st = os.lstat(path, dir_fd=dir_fd) - except Exception as err: + except OSError as err: onexc(os.lstat, path, err) return try: fd = os.open(path, os.O_RDONLY, dir_fd=dir_fd) fd_closed = False - except Exception as err: + except OSError as err: onexc(os.open, path, err) return try: @@ -769,7 +795,12 @@ def onexc(*args): _rmtree_safe_fd(fd, path, onexc) try: os.close(fd) + except OSError as err: + # close() should not be retried after an error. fd_closed = True + onexc(os.close, path, err) + fd_closed = True + try: os.rmdir(path, dir_fd=dir_fd) except OSError as err: onexc(os.rmdir, path, err) @@ -781,12 +812,20 @@ def onexc(*args): onexc(os.path.islink, path, err) finally: if not fd_closed: - os.close(fd) + try: + os.close(fd) + except OSError as err: + onexc(os.close, path, err) else: if dir_fd is not None: raise NotImplementedError("dir_fd unavailable on this platform") try: - if _rmtree_islink(path): + st = os.lstat(path) + except OSError as err: + onexc(os.lstat, path, err) + return + try: + if _rmtree_islink(st): # symlinks to directories are forbidden, see bug #1669 raise OSError("Cannot call rmtree on a symbolic link") except OSError as err: @@ -846,7 +885,7 @@ def move(src, dst, copy_function=copy2): sys.audit("shutil.move", src, dst) real_dst = dst if os.path.isdir(dst): - if _samefile(src, dst): + if _samefile(src, dst) and not os.path.islink(src): # We might be on a case insensitive filesystem, # perform the rename anyway. os.rename(src, dst) diff --git a/Lib/signal.py b/Lib/signal.py index 50b215b29d2fad..c8cd3d4f597ca5 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -22,9 +22,11 @@ def _int_to_enum(value, enum_klass): - """Convert a numeric value to an IntEnum member. - If it's not a known member, return the numeric value itself. + """Convert a possible numeric value to an IntEnum member. + If it's not a known member, return the value itself. """ + if not isinstance(value, int): + return value try: return enum_klass(value) except ValueError: diff --git a/Lib/site.py b/Lib/site.py index 672fa7b000ad02..6f5738b02cb23b 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -260,6 +260,10 @@ def check_enableusersite(): # # See https://bugs.python.org/issue29585 +# Copy of sysconfig._get_implementation() +def _get_implementation(): + return 'Python' + # Copy of sysconfig._getuserbase() def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) @@ -275,7 +279,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -288,12 +292,14 @@ def joinuser(*args): def _get_path(userbase): version = sys.version_info + implementation = _get_implementation() + implementation_lower = implementation.lower() if os.name == 'nt': ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\Python{ver_nodot}\\site-packages' + return f'{userbase}\\{implementation}{ver_nodot}\\site-packages' if sys.platform == 'darwin' and sys._framework: - return f'{userbase}/lib/python/site-packages' + return f'{userbase}/lib/{implementation_lower}/site-packages' return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' @@ -361,6 +367,8 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) + implementation = _get_implementation().lower() + ver = sys.version_info if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -368,7 +376,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - "python%d.%d" % sys.version_info[:2], + f"{implementation}{ver[0]}.{ver[1]}", "site-packages") sitepackages.append(path) else: @@ -444,8 +452,7 @@ def register_readline(): # Reading the initialization (config) file may not be enough to set a # completion key, so we set one first and then read the file. - readline_doc = getattr(readline, '__doc__', '') - if readline_doc is not None and 'libedit' in readline_doc: + if readline.backend == 'editline': readline.parse_and_bind('bind ^I rl_complete') else: readline.parse_and_bind('tab: complete') diff --git a/Lib/ssl.py b/Lib/ssl.py index 62e55857141dfc..d01484964b6895 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -116,7 +116,7 @@ from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3 + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION @@ -1270,10 +1270,14 @@ def recv(self, buflen=1024, flags=0): def recv_into(self, buffer, nbytes=None, flags=0): self._checkClosed() - if buffer and (nbytes is None): - nbytes = len(buffer) - elif nbytes is None: - nbytes = 1024 + if nbytes is None: + if buffer is not None: + with memoryview(buffer) as view: + nbytes = view.nbytes + if not nbytes: + nbytes = 1024 + else: + nbytes = 1024 if self._sslobj is not None: if flags != 0: raise ValueError( diff --git a/Lib/statistics.py b/Lib/statistics.py index 4da06889c6db46..83aaedb04515e0 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -527,8 +527,10 @@ def count(iterable): def geometric_mean(data): """Convert data to floats and compute the geometric mean. - Raises a StatisticsError if the input dataset is empty, - if it contains a zero, or if it contains a negative value. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. + + Returns zero if the product of inputs is zero. No special efforts are made to achieve exact results. (However, this may change in the future.) @@ -536,11 +538,25 @@ def geometric_mean(data): >>> round(geometric_mean([54, 24, 36]), 9) 36.0 """ - try: - return exp(fmean(map(log, data))) - except ValueError: - raise StatisticsError('geometric mean requires a non-empty dataset ' - 'containing positive numbers') from None + n = 0 + found_zero = False + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 + return exp(total / n) def harmonic_mean(data, weights=None): diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 6df5dd551ea67e..d5bd9a9e31aa04 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -74,8 +74,8 @@ else: _mswindows = True -# wasm32-emscripten and wasm32-wasi do not support processes -_can_fork_exec = sys.platform not in {"emscripten", "wasi"} +# some platforms do not support subprocesses +_can_fork_exec = sys.platform not in {"emscripten", "wasi", "ios", "tvos", "watchos"} if _mswindows: import _winapi @@ -103,18 +103,22 @@ if _can_fork_exec: from _posixsubprocess import fork_exec as _fork_exec # used in methods that are called by __del__ - _waitpid = os.waitpid - _waitstatus_to_exitcode = os.waitstatus_to_exitcode - _WIFSTOPPED = os.WIFSTOPPED - _WSTOPSIG = os.WSTOPSIG - _WNOHANG = os.WNOHANG + class _del_safe: + waitpid = os.waitpid + waitstatus_to_exitcode = os.waitstatus_to_exitcode + WIFSTOPPED = os.WIFSTOPPED + WSTOPSIG = os.WSTOPSIG + WNOHANG = os.WNOHANG + ECHILD = errno.ECHILD else: - _fork_exec = None - _waitpid = None - _waitstatus_to_exitcode = None - _WIFSTOPPED = None - _WSTOPSIG = None - _WNOHANG = None + class _del_safe: + waitpid = None + waitstatus_to_exitcode = None + WIFSTOPPED = None + WSTOPSIG = None + WNOHANG = None + ECHILD = errno.ECHILD + import select import selectors @@ -744,6 +748,7 @@ def _use_posix_spawn(): # guarantee the given libc/syscall API will be used. _USE_POSIX_SPAWN = _use_posix_spawn() _USE_VFORK = True +_HAVE_POSIX_SPAWN_CLOSEFROM = hasattr(os, 'POSIX_SPAWN_CLOSEFROM') class Popen: @@ -1747,14 +1752,11 @@ def _get_handles(self, stdin, stdout, stderr): errread, errwrite) - def _posix_spawn(self, args, executable, env, restore_signals, + def _posix_spawn(self, args, executable, env, restore_signals, close_fds, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program using os.posix_spawn().""" - if env is None: - env = os.environ - kwargs = {} if restore_signals: # See _Py_RestoreSignals() in Python/pylifecycle.c @@ -1776,6 +1778,10 @@ def _posix_spawn(self, args, executable, env, restore_signals, ): if fd != -1: file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2)) + + if close_fds: + file_actions.append((os.POSIX_SPAWN_CLOSEFROM, 3)) + if file_actions: kwargs['file_actions'] = file_actions @@ -1823,7 +1829,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, if (_USE_POSIX_SPAWN and os.path.dirname(executable) and preexec_fn is None - and not close_fds + and (not close_fds or _HAVE_POSIX_SPAWN_CLOSEFROM) and not pass_fds and cwd is None and (p2cread == -1 or p2cread > 2) @@ -1835,7 +1841,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, and gids is None and uid is None and umask < 0): - self._posix_spawn(args, executable, env, restore_signals, + self._posix_spawn(args, executable, env, restore_signals, close_fds, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) @@ -1951,20 +1957,16 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, raise child_exception_type(err_msg) - def _handle_exitstatus(self, sts, - _waitstatus_to_exitcode=_waitstatus_to_exitcode, - _WIFSTOPPED=_WIFSTOPPED, - _WSTOPSIG=_WSTOPSIG): + def _handle_exitstatus(self, sts, _del_safe=_del_safe): """All callers to this function MUST hold self._waitpid_lock.""" # This method is called (indirectly) by __del__, so it cannot # refer to anything outside of its local scope. - if _WIFSTOPPED(sts): - self.returncode = -_WSTOPSIG(sts) + if _del_safe.WIFSTOPPED(sts): + self.returncode = -_del_safe.WSTOPSIG(sts) else: - self.returncode = _waitstatus_to_exitcode(sts) + self.returncode = _del_safe.waitstatus_to_exitcode(sts) - def _internal_poll(self, _deadstate=None, _waitpid=_waitpid, - _WNOHANG=_WNOHANG, _ECHILD=errno.ECHILD): + def _internal_poll(self, _deadstate=None, _del_safe=_del_safe): """Check if child process has terminated. Returns returncode attribute. @@ -1980,13 +1982,13 @@ def _internal_poll(self, _deadstate=None, _waitpid=_waitpid, try: if self.returncode is not None: return self.returncode # Another thread waited. - pid, sts = _waitpid(self.pid, _WNOHANG) + pid, sts = _del_safe.waitpid(self.pid, _del_safe.WNOHANG) if pid == self.pid: self._handle_exitstatus(sts) except OSError as e: if _deadstate is not None: self.returncode = _deadstate - elif e.errno == _ECHILD: + elif e.errno == _del_safe.ECHILD: # This happens if SIGCLD is set to be ignored or # waiting for child processes has otherwise been # disabled for our process. This child is dead, we diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 2a7fa45be079de..07ab27c7fb0c35 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -26,24 +26,24 @@ _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, 'posix_home': { - 'stdlib': '{installed_base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{installed_base}/include/python', - 'platinclude': '{installed_base}/include/python', + 'stdlib': '{installed_base}/lib/{implementation_lower}', + 'platstdlib': '{base}/lib/{implementation_lower}', + 'purelib': '{base}/lib/{implementation_lower}', + 'platlib': '{base}/lib/{implementation_lower}', + 'include': '{installed_base}/include/{implementation_lower}', + 'platinclude': '{installed_base}/include/{implementation_lower}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -75,14 +75,14 @@ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -104,6 +104,8 @@ else: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] +def _get_implementation(): + return 'Python' # NOTE: site.py has copy of this function. # Sync it when modify this function. @@ -121,7 +123,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -135,29 +137,29 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'stdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'platstdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'purelib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/{implementation}{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/{implementation}{py_version_nodot_plat}/Scripts', 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/{implementation_lower}', + 'platstdlib': '{userbase}/lib/{implementation_lower}', + 'purelib': '{userbase}/lib/{implementation_lower}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -404,7 +406,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_PROJECT_BASE, "PC") + inc_dir = os.path.dirname(sys._base_executable) else: inc_dir = _PROJECT_BASE else: @@ -459,6 +461,8 @@ def _init_config_vars(): _CONFIG_VARS['platbase'] = _EXEC_PREFIX _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir + _CONFIG_VARS['implementation'] = _get_implementation() + _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() try: _CONFIG_VARS['abiflags'] = sys.abiflags except AttributeError: diff --git a/Lib/tarfile.py b/Lib/tarfile.py index ec32f9ba49b03f..20e0394507f5db 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -330,10 +330,11 @@ def write(self, s): class _Stream: """Class that serves as an adapter between TarFile and a stream-like object. The stream-like object only - needs to have a read() or write() method and is accessed - blockwise. Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin, - sys.stdout, a socket, a tape device etc. + needs to have a read() or write() method that works with bytes, + and the method is accessed blockwise. + Use of gzip or bzip2 compression is possible. + A stream-like object could be for example: sys.stdin.buffer, + sys.stdout.buffer, a socket, a tape device etc. _Stream is intended to be used only internally. """ @@ -2106,6 +2107,10 @@ def list(self, verbose=True, *, members=None): output is produced. `members' is optional and must be a subset of the list returned by getmembers(). """ + # Convert tarinfo type to stat type. + type2mode = {REGTYPE: stat.S_IFREG, SYMTYPE: stat.S_IFLNK, + FIFOTYPE: stat.S_IFIFO, CHRTYPE: stat.S_IFCHR, + DIRTYPE: stat.S_IFDIR, BLKTYPE: stat.S_IFBLK} self._check() if members is None: @@ -2115,7 +2120,8 @@ def list(self, verbose=True, *, members=None): if tarinfo.mode is None: _safe_print("??????????") else: - _safe_print(stat.filemode(tarinfo.mode)) + modetype = type2mode.get(tarinfo.type, 0) + _safe_print(stat.filemode(modetype | tarinfo.mode)) _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, tarinfo.gname or tarinfo.gid)) if tarinfo.ischr() or tarinfo.isblk(): diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 2b4f4313247128..b5a15f7b72c872 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -269,6 +269,22 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type): raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") +def _dont_follow_symlinks(func, path, *args): + # Pass follow_symlinks=False, unless not supported on this platform. + if func in _os.supports_follow_symlinks: + func(path, *args, follow_symlinks=False) + elif not _os.path.islink(path): + func(path, *args) + +def _resetperms(path): + try: + chflags = _os.chflags + except AttributeError: + pass + else: + _dont_follow_symlinks(chflags, path, 0) + _dont_follow_symlinks(_os.chmod, path, 0o700) + # User visible interfaces. @@ -872,26 +888,37 @@ def __init__(self, suffix=None, prefix=None, dir=None, ignore_errors=self._ignore_cleanup_errors, delete=self._delete) @classmethod - def _rmtree(cls, name, ignore_errors=False): + def _rmtree(cls, name, ignore_errors=False, repeated=False): def onexc(func, path, exc): if isinstance(exc, PermissionError): - def resetperms(path): - try: - _os.chflags(path, 0) - except AttributeError: - pass - _os.chmod(path, 0o700) + if repeated and path == name: + if ignore_errors: + return + raise try: if path != name: - resetperms(_os.path.dirname(path)) - resetperms(path) + _resetperms(_os.path.dirname(path)) + _resetperms(path) try: _os.unlink(path) - # PermissionError is raised on FreeBSD for directories - except (IsADirectoryError, PermissionError): + except IsADirectoryError: cls._rmtree(path, ignore_errors=ignore_errors) + except PermissionError: + # The PermissionError handler was originally added for + # FreeBSD in directories, but it seems that it is raised + # on Windows too. + # bpo-43153: Calling _rmtree again may + # raise NotADirectoryError and mask the PermissionError. + # So we must re-raise the current PermissionError if + # path is not a directory. + if not _os.path.isdir(path) or _os.path.isjunction(path): + if ignore_errors: + return + raise + cls._rmtree(path, ignore_errors=ignore_errors, + repeated=(path == name)) except FileNotFoundError: pass elif isinstance(exc, FileNotFoundError): diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index ec003d8dc4314d..8e4e0765d46809 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4455,6 +4455,59 @@ def test_shared_memory_cleaned_after_process_termination(self): "resource_tracker: There appear to be 1 leaked " "shared_memory objects to clean up at shutdown", err) + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + def test_shared_memory_untracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=False, it must not cause the memory to be deleted + # when terminating. + cmd = '''if 1: + import sys + from multiprocessing.shared_memory import SharedMemory + mem = SharedMemory(create=False, name=sys.argv[1], track=False) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + # The resource tracker shares pipes with the subprocess, and so + # err existing means that the tracker process has terminated now. + try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertNotIn(b"resource_tracker", err) + self.assertEqual(rc, 0) + mem2 = shared_memory.SharedMemory(create=False, name=mem.name) + mem2.close() + finally: + try: + mem.unlink() + except OSError: + pass + mem.close() + + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + def test_shared_memory_tracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=True, it must cause the memory to be deleted when + # terminating. + cmd = '''if 1: + import sys + from multiprocessing.shared_memory import SharedMemory + mem = SharedMemory(create=False, name=sys.argv[1], track=True) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertEqual(rc, 0) + self.assertIn( + b"resource_tracker: There appear to be 1 leaked " + b"shared_memory objects to clean up at shutdown", err) + finally: + try: + mem.unlink() + except OSError: + pass + resource_tracker.unregister(mem._name, "shared_memory") + mem.close() + # # Test to verify that `Finalize` works. # @@ -4671,6 +4724,29 @@ def test_level(self): root_logger.setLevel(root_level) logger.setLevel(level=LOG_LEVEL) + def test_filename(self): + logger = multiprocessing.get_logger() + original_level = logger.level + try: + logger.setLevel(util.DEBUG) + stream = io.StringIO() + handler = logging.StreamHandler(stream) + logging_format = '[%(levelname)s] [%(filename)s] %(message)s' + handler.setFormatter(logging.Formatter(logging_format)) + logger.addHandler(handler) + logger.info('1') + util.info('2') + logger.debug('3') + filename = os.path.basename(__file__) + log_record = stream.getvalue() + self.assertIn(f'[INFO] [{filename}] 1', log_record) + self.assertIn(f'[INFO] [{filename}] 2', log_record) + self.assertIn(f'[DEBUG] [{filename}] 3', log_record) + finally: + logger.setLevel(original_level) + logger.removeHandler(handler) + handler.close() + # class _TestLoggingProcessName(BaseTestCase): # diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 81f88c4d1535ce..b15aeb898d35a1 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4951,6 +4951,65 @@ static PyObject * Test_meth_coexist_impl(TestObj *self) /*[clinic end generated code: output=808a293d0cd27439 input=2a1d75b5e6fec6dd]*/ +/*[clinic input] +@getter +Test.property +[clinic start generated code]*/ + +#if defined(Test_property_HAS_DOCSTR) +# define Test_property_DOCSTR Test_property__doc__ +#else +# define Test_property_DOCSTR NULL +#endif +#if defined(TEST_PROPERTY_GETSETDEF) +# undef TEST_PROPERTY_GETSETDEF +# define TEST_PROPERTY_GETSETDEF {"property", (getter)Test_property_get, (setter)Test_property_set, Test_property_DOCSTR}, +#else +# define TEST_PROPERTY_GETSETDEF {"property", (getter)Test_property_get, NULL, Test_property_DOCSTR}, +#endif + +static PyObject * +Test_property_get_impl(TestObj *self); + +static PyObject * +Test_property_get(TestObj *self, void *Py_UNUSED(context)) +{ + return Test_property_get_impl(self); +} + +static PyObject * +Test_property_get_impl(TestObj *self) +/*[clinic end generated code: output=27b519719d992e03 input=2d92b3449fbc7d2b]*/ + +/*[clinic input] +@setter +Test.property +[clinic start generated code]*/ + +#if defined(TEST_PROPERTY_HAS_DOCSTR) +# define Test_property_DOCSTR Test_property__doc__ +#else +# define Test_property_DOCSTR NULL +#endif +#if defined(TEST_PROPERTY_GETSETDEF) +# undef TEST_PROPERTY_GETSETDEF +# define TEST_PROPERTY_GETSETDEF {"property", (getter)Test_property_get, (setter)Test_property_set, Test_property_DOCSTR}, +#else +# define TEST_PROPERTY_GETSETDEF {"property", NULL, (setter)Test_property_set, NULL}, +#endif + +static int +Test_property_set_impl(TestObj *self, PyObject *value); + +static int +Test_property_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) +{ + return Test_property_set_impl(self, value); +} + +static int +Test_property_set_impl(TestObj *self, PyObject *value) +/*[clinic end generated code: output=9797cd03c5204ddb input=3bc3f46a23c83a88]*/ /*[clinic input] output push diff --git a/Lib/test/doctest_lineno.py b/Lib/test/doctest_lineno.py index 729a68aceaa990..677c569cf710eb 100644 --- a/Lib/test/doctest_lineno.py +++ b/Lib/test/doctest_lineno.py @@ -49,5 +49,21 @@ def method_with_doctest(self): 'method_with_doctest' """ + @classmethod + def classmethod_with_doctest(cls): + """ + This has a doctest! + >>> MethodWrapper.classmethod_with_doctest.__name__ + 'classmethod_with_doctest' + """ + + @property + def property_with_doctest(self): + """ + This has a doctest! + >>> MethodWrapper.property_with_doctest.__name__ + 'property_with_doctest' + """ + # https://github.com/python/cpython/issues/99433 str_wrapper = object().__str__ diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index a5f02d6335f58f..0053bce4292f64 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -3,7 +3,7 @@ import shlex import sys from test.support import os_helper, Py_DEBUG -from .utils import ALL_RESOURCES, RESOURCE_NAMES +from .utils import ALL_RESOURCES, RESOURCE_NAMES, TestFilter USAGE = """\ @@ -161,7 +161,7 @@ def __init__(self, **kwargs) -> None: self.forever = False self.header = False self.failfast = False - self.match_tests = [] + self.match_tests: TestFilter = [] self.pgo = False self.pgo_extended = False self.worker_json = None diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 86428945a6def2..7ca1b1cb65ae40 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -295,7 +295,9 @@ def run_test( namespace = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=namespace) result = namespace['result'] - result.covered_lines = list(tracer.counts) + # Mypy doesn't know about this attribute yet, + # but it will do soon: https://github.com/python/typeshed/pull/11091 + result.covered_lines = list(tracer.counts) # type: ignore[attr-defined] else: result = run_single_test(test_name, runtests) @@ -309,7 +311,7 @@ def run_tests_sequentially(self, runtests) -> None: else: tracer = None - save_modules = sys.modules.keys() + save_modules = set(sys.modules) jobs = runtests.get_jobs() if jobs is not None: @@ -333,10 +335,18 @@ def run_tests_sequentially(self, runtests) -> None: result = self.run_test(test_name, runtests, tracer) - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) + # Unload the newly imported test modules (best effort finalization) + new_modules = [module for module in sys.modules + if module not in save_modules and + module.startswith(("test.", "test_"))] + for module in new_modules: + sys.modules.pop(module, None) + # Remove the attribute of the parent module. + parent, _, name = module.rpartition('.') + try: + delattr(sys.modules[parent], name) + except (KeyError, AttributeError): + pass if result.must_stop(self.fail_fast, self.fail_env_changed): break @@ -371,7 +381,8 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: os.unlink(self.next_single_filename) if coverage is not None: - coverage.write_results(show_missing=True, summary=True, + # uses a new-in-Python 3.13 keyword argument that mypy doesn't know about yet: + coverage.write_results(show_missing=True, summary=True, # type: ignore[call-arg] coverdir=self.coverage_dir, ignore_missing_files=True) @@ -420,7 +431,6 @@ def create_run_tests(self, tests: TestTuple): python_cmd=self.python_cmd, randomize=self.randomize, random_seed=self.random_seed, - json_file=None, ) def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: @@ -432,7 +442,10 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if self.num_workers < 0: # Use all CPUs + 2 extra worker processes for tests # that like to sleep - self.num_workers = (os.process_cpu_count() or 1) + 2 + # + # os.process.cpu_count() is new in Python 3.13; + # mypy doesn't know about it yet + self.num_workers = (os.process_cpu_count() or 1) + 2 # type: ignore[attr-defined] # For a partial run, we do not need to clutter the output. if (self.want_header diff --git a/Lib/test/libregrtest/mypy.ini b/Lib/test/libregrtest/mypy.ini index fefc347728a701..22c7c7a9acef14 100644 --- a/Lib/test/libregrtest/mypy.ini +++ b/Lib/test/libregrtest/mypy.ini @@ -5,7 +5,7 @@ [mypy] files = Lib/test/libregrtest explicit_package_bases = True -python_version = 3.11 +python_version = 3.12 platform = linux pretty = True @@ -25,7 +25,7 @@ warn_return_any = False disable_error_code = return # Enable --strict-optional for these ASAP: -[mypy-Lib.test.libregrtest.main.*,Lib.test.libregrtest.run_workers.*,Lib.test.libregrtest.worker.*,Lib.test.libregrtest.single.*,Lib.test.libregrtest.results.*,Lib.test.libregrtest.utils.*] +[mypy-Lib.test.libregrtest.main.*,Lib.test.libregrtest.run_workers.*] strict_optional = False # Various internal modules that typeshed deliberately doesn't have stubs for: diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index ada1a65b867ee6..5836a8421cb42d 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -52,7 +52,8 @@ def runtest_refleak(test_name, test_func, except ImportError: zdc = None # Run unmodified on platforms without zipimport support else: - zdc = zipimport._zip_directory_cache.copy() + # private attribute that mypy doesn't know about: + zdc = zipimport._zip_directory_cache.copy() # type: ignore[attr-defined] abcs = {} for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 71aaef3ae9ae61..a41ea8aba028c3 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -34,7 +34,7 @@ def __init__(self): self.test_times: list[tuple[float, TestName]] = [] self.stats = TestStats() # used by --junit-xml - self.testsuite_xml: list[str] = [] + self.testsuite_xml: list = [] # used by -T with -j self.covered_lines: set[Location] = set() @@ -117,6 +117,8 @@ def accumulate_result(self, result: TestResult, runtests: RunTests): self.worker_bug = True if result.has_meaningful_duration() and not rerun: + if result.duration is None: + raise ValueError("result.duration is None") self.test_times.append((result.duration, test_name)) if result.stats is not None: self.stats.accumulate(result.stats) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 99c2cf34d206d0..18a0342f0611cf 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -10,7 +10,7 @@ import threading import time import traceback -from typing import Literal, TextIO +from typing import Any, Literal, TextIO from test import support from test.support import os_helper, MS_WINDOWS @@ -18,7 +18,7 @@ from .logger import Logger from .result import TestResult, State from .results import TestResults -from .runtests import RunTests, JsonFile, JsonFileType +from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, @@ -162,7 +162,7 @@ def stop(self) -> None: self._stopped = True self._kill() - def _run_process(self, runtests: RunTests, output_fd: int, + def _run_process(self, runtests: WorkerRunTests, output_fd: int, tmp_dir: StrPath | None = None) -> int | None: popen = create_worker_process(runtests, output_fd, tmp_dir) self._popen = popen @@ -243,34 +243,34 @@ def create_json_file(self, stack: contextlib.ExitStack) -> tuple[JsonFile, TextI json_fd = json_tmpfile.fileno() if MS_WINDOWS: - json_handle = msvcrt.get_osfhandle(json_fd) + # The msvcrt module is only available on Windows; + # we run mypy with `--platform=linux` in CI + json_handle: int = msvcrt.get_osfhandle(json_fd) # type: ignore[attr-defined] json_file = JsonFile(json_handle, JsonFileType.WINDOWS_HANDLE) else: json_file = JsonFile(json_fd, JsonFileType.UNIX_FD) return (json_file, json_tmpfile) - def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> RunTests: - """Create the worker RunTests.""" - + def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> WorkerRunTests: tests = (test_name,) if self.runtests.rerun: match_tests = self.runtests.get_match_tests(test_name) else: match_tests = None - kwargs = {} + kwargs: dict[str, Any] = {} if match_tests: kwargs['match_tests'] = [(test, True) for test in match_tests] if self.runtests.output_on_failure: kwargs['verbose'] = True kwargs['output_on_failure'] = False - return self.runtests.copy( + return self.runtests.create_worker_runtests( tests=tests, json_file=json_file, **kwargs) - def run_tmp_files(self, worker_runtests: RunTests, + def run_tmp_files(self, worker_runtests: WorkerRunTests, stdout_fd: int) -> tuple[int | None, list[StrPath]]: # gh-93353: Check for leaked temporary files in the parent process, # since the deletion of temporary files can happen late during @@ -345,6 +345,7 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: json_file, json_tmpfile = self.create_json_file(stack) worker_runtests = self.create_worker_runtests(test_name, json_file) + retcode: str | int | None retcode, tmp_files = self.run_tmp_files(worker_runtests, stdout_file.fileno()) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index ac47c07f8d4341..edd72276320e41 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -33,7 +33,8 @@ def configure_subprocess(self, popen_kwargs: dict) -> None: popen_kwargs['pass_fds'] = [self.file] case JsonFileType.WINDOWS_HANDLE: # Windows handle - startupinfo = subprocess.STARTUPINFO() + # We run mypy with `--platform=linux` so it complains about this: + startupinfo = subprocess.STARTUPINFO() # type: ignore[attr-defined] startupinfo.lpAttributeList = {"handle_list": [self.file]} popen_kwargs['startupinfo'] = startupinfo @@ -92,13 +93,17 @@ class RunTests: python_cmd: tuple[str, ...] | None randomize: bool random_seed: int | str - json_file: JsonFile | None - def copy(self, **override): + def copy(self, **override) -> 'RunTests': state = dataclasses.asdict(self) state.update(override) return RunTests(**state) + def create_worker_runtests(self, **override): + state = dataclasses.asdict(self) + state.update(override) + return WorkerRunTests(**state) + def get_match_tests(self, test_name) -> FilterTuple | None: if self.match_tests_dict is not None: return self.match_tests_dict.get(test_name, None) @@ -119,13 +124,6 @@ def iter_tests(self): else: yield from self.tests - def as_json(self) -> StrJSON: - return json.dumps(self, cls=_EncodeRunTests) - - @staticmethod - def from_json(worker_json: StrJSON) -> 'RunTests': - return json.loads(worker_json, object_hook=_decode_runtests) - def json_file_use_stdout(self) -> bool: # Use STDOUT in two cases: # @@ -140,9 +138,21 @@ def json_file_use_stdout(self) -> bool: ) +@dataclasses.dataclass(slots=True, frozen=True) +class WorkerRunTests(RunTests): + json_file: JsonFile + + def as_json(self) -> StrJSON: + return json.dumps(self, cls=_EncodeRunTests) + + @staticmethod + def from_json(worker_json: StrJSON) -> 'WorkerRunTests': + return json.loads(worker_json, object_hook=_decode_runtests) + + class _EncodeRunTests(json.JSONEncoder): def default(self, o: Any) -> dict[str, Any]: - if isinstance(o, RunTests): + if isinstance(o, WorkerRunTests): result = dataclasses.asdict(o) result["__runtests__"] = True return result @@ -157,6 +167,6 @@ def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak']) if data['json_file']: data['json_file'] = JsonFile(**data['json_file']) - return RunTests(**data) + return WorkerRunTests(**data) else: return data diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 97edba9f87d7f9..9e9741493e9a5b 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -124,7 +124,8 @@ def setup_tests(runtests: RunTests): support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) if runtests.hunt_refleak: - unittest.BaseTestSuite._cleanup = False + # private attribute that mypy doesn't know about: + unittest.BaseTestSuite._cleanup = False # type: ignore[attr-defined] if runtests.gc_threshold is not None: gc.set_threshold(runtests.gc_threshold) diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 5c7bc7d40fb394..235029d8620ff5 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -122,10 +122,6 @@ def _load_run_test(result: TestResult, runtests: RunTests) -> None: # Load the test module and run the tests. test_name = result.test_name module_name = abs_module_name(test_name, runtests.test_dir) - - # Remove the module from sys.module to reload it if it was already imported - sys.modules.pop(module_name, None) - test_mod = importlib.import_module(module_name) if hasattr(test_mod, "test_main"): @@ -237,11 +233,11 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: output_on_failure = runtests.output_on_failure timeout = runtests.timeout - use_timeout = ( - timeout is not None and threading_helper.can_start_thread - ) - if use_timeout: + if timeout is not None and threading_helper.can_start_thread: + use_timeout = True faulthandler.dump_traceback_later(timeout, exit=True) + else: + use_timeout = False try: setup_tests(runtests) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index e4a28af381ee2d..b30025d962413c 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -12,7 +12,7 @@ import sysconfig import tempfile import textwrap -from collections.abc import Callable +from collections.abc import Callable, Iterable from test import support from test.support import os_helper @@ -291,7 +291,7 @@ def get_build_info(): # --disable-gil if sysconfig.get_config_var('Py_GIL_DISABLED'): - build.append("nogil") + build.append("free_threading") if hasattr(sys, 'gettotalrefcount'): # --with-pydebug @@ -340,6 +340,9 @@ def get_build_info(): # --with-undefined-behavior-sanitizer if support.check_sanitizer(ub=True): sanitizers.append("UBSAN") + # --with-thread-sanitizer + if support.check_sanitizer(thread=True): + sanitizers.append("TSAN") if sanitizers: build.append('+'.join(sanitizers)) @@ -377,10 +380,19 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: # Python out of the source tree, especially when the # source tree is read only. tmp_dir = sysconfig.get_config_var('srcdir') + if not tmp_dir: + raise RuntimeError( + "Could not determine the correct value for tmp_dir" + ) tmp_dir = os.path.join(tmp_dir, 'build') else: # WASI platform tmp_dir = sysconfig.get_config_var('projectbase') + if not tmp_dir: + raise RuntimeError( + "sysconfig.get_config_var('projectbase') " + f"unexpectedly returned {tmp_dir!r} on WASI" + ) tmp_dir = os.path.join(tmp_dir, 'build') # When get_temp_dir() is called in a worker process, @@ -547,7 +559,7 @@ def is_cross_compiled(): return ('_PYTHON_HOST_PLATFORM' in os.environ) -def format_resources(use_resources: tuple[str, ...]): +def format_resources(use_resources: Iterable[str]): use_resources = set(use_resources) all_resources = set(ALL_RESOURCES) @@ -580,9 +592,10 @@ def display_header(use_resources: tuple[str, ...], print("== Python build:", ' '.join(get_build_info())) print("== cwd:", os.getcwd()) - cpu_count = os.cpu_count() + cpu_count: object = os.cpu_count() if cpu_count: - process_cpu_count = os.process_cpu_count() + # The function is new in Python 3.13; mypy doesn't know about it yet: + process_cpu_count = os.process_cpu_count() # type: ignore[attr-defined] if process_cpu_count and process_cpu_count != cpu_count: cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)" print("== CPU count:", cpu_count) @@ -624,6 +637,7 @@ def display_header(use_resources: tuple[str, ...], asan = support.check_sanitizer(address=True) msan = support.check_sanitizer(memory=True) ubsan = support.check_sanitizer(ub=True) + tsan = support.check_sanitizer(thread=True) sanitizers = [] if asan: sanitizers.append("address") @@ -631,12 +645,15 @@ def display_header(use_resources: tuple[str, ...], sanitizers.append("memory") if ubsan: sanitizers.append("undefined behavior") + if tsan: + sanitizers.append("thread") if sanitizers: print(f"== sanitizers: {', '.join(sanitizers)}") for sanitizer, env_var in ( (asan, "ASAN_OPTIONS"), (msan, "MSAN_OPTIONS"), (ubsan, "UBSAN_OPTIONS"), + (tsan, "TSAN_OPTIONS"), ): options= os.environ.get(env_var) if sanitizer and options is not None: diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index b3bb0b7f34a060..7a6d33d4499943 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -7,7 +7,7 @@ from test.support import os_helper, Py_DEBUG from .setup import setup_process, setup_test_dir -from .runtests import RunTests, JsonFile, JsonFileType +from .runtests import WorkerRunTests, JsonFile, JsonFileType from .single import run_single_test from .utils import ( StrPath, StrJSON, TestFilter, @@ -17,7 +17,7 @@ USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) -def create_worker_process(runtests: RunTests, output_fd: int, +def create_worker_process(runtests: WorkerRunTests, output_fd: int, tmp_dir: StrPath | None = None) -> subprocess.Popen: python_cmd = runtests.python_cmd worker_json = runtests.as_json() @@ -73,7 +73,7 @@ def create_worker_process(runtests: RunTests, output_fd: int, def worker_process(worker_json: StrJSON) -> NoReturn: - runtests = RunTests.from_json(worker_json) + runtests = WorkerRunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: TestFilter = runtests.match_tests json_file: JsonFile = runtests.json_file diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index ddb180ef5ef825..74b82caf742f20 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1825,6 +1825,14 @@ def test_unicode_high_plane(self): t2 = self.loads(p) self.assert_is_copy(t, t2) + def test_unicode_memoization(self): + # Repeated str is re-used (even when escapes added). + for proto in protocols: + for s in '', 'xyz', 'xyz\n', 'x\\yz', 'x\xa1yz\r': + p = self.dumps((s, s), proto) + s1, s2 = self.loads(p) + self.assertIs(s1, s2) + def test_bytes(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: @@ -3514,6 +3522,84 @@ def __init__(self): pass self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) + def test_unpickler_bad_file(self): + # bpo-38384: Crash in _pickle if the read attribute raises an error. + def raises_oserror(self, *args, **kwargs): + raise OSError + @property + def bad_property(self): + 1/0 + + # File without read and readline + class F: + pass + self.assertRaises((AttributeError, TypeError), self.Unpickler, F()) + + # File without read + class F: + readline = raises_oserror + self.assertRaises((AttributeError, TypeError), self.Unpickler, F()) + + # File without readline + class F: + read = raises_oserror + self.assertRaises((AttributeError, TypeError), self.Unpickler, F()) + + # File with bad read + class F: + read = bad_property + readline = raises_oserror + self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + + # File with bad readline + class F: + readline = bad_property + read = raises_oserror + self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + + # File with bad readline, no read + class F: + readline = bad_property + self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + + # File with bad read, no readline + class F: + read = bad_property + self.assertRaises((AttributeError, ZeroDivisionError), self.Unpickler, F()) + + # File with bad peek + class F: + peek = bad_property + read = raises_oserror + readline = raises_oserror + try: + self.Unpickler(F()) + except ZeroDivisionError: + pass + + # File with bad readinto + class F: + readinto = bad_property + read = raises_oserror + readline = raises_oserror + try: + self.Unpickler(F()) + except ZeroDivisionError: + pass + + def test_pickler_bad_file(self): + # File without write + class F: + pass + self.assertRaises(TypeError, self.Pickler, F()) + + # File with bad write + class F: + @property + def write(self): + 1/0 + self.assertRaises(ZeroDivisionError, self.Pickler, F()) + def check_dumps_loads_oob_buffers(self, dumps, loads): # No need to do the full gamut of tests here, just enough to # check that dumps() and loads() redirect their arguments diff --git a/Lib/test/pydocfodder.py b/Lib/test/pydocfodder.py index d0750e5a43c0c0..a3ef2231243954 100644 --- a/Lib/test/pydocfodder.py +++ b/Lib/test/pydocfodder.py @@ -2,8 +2,8 @@ import types -class A_new: - "A new-style class." +class A: + "A class." def A_method(self): "Method defined in A." @@ -41,8 +41,8 @@ def _delx(self): A_int_alias = int -class B_new(A_new): - "A new-style class, derived from A_new." +class B(A): + "A class, derived from A." def AB_method(self): "Method defined in A and B." @@ -61,8 +61,8 @@ def BD_method(self): def BCD_method(self): "Method defined in B, C and D." -class C_new(A_new): - "A new-style class, derived from A_new." +class C(A): + "A class, derived from A." def AC_method(self): "Method defined in A and C." @@ -81,8 +81,8 @@ def C_method(self): def CD_method(self): "Method defined in C and D." -class D_new(B_new, C_new): - """A new-style class, derived from B_new and C_new. +class D(B, C): + """A class, derived from B and C. """ def AD_method(self): diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 49e41ca6cdaf98..6dfb7f37e406a5 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -925,6 +925,8 @@ def collect_windows(info_add): stderr=subprocess.PIPE, text=True) output = proc.communicate()[0] + if proc.returncode == 0xc0000142: + return if proc.returncode: output = "" except OSError: diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py new file mode 100644 index 00000000000000..9c3d0c7cf4bfaa --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py @@ -0,0 +1,11 @@ +import sys +import unittest +import test_regrtest_b.util + +class Test(unittest.TestCase): + def test(self): + test_regrtest_b.util # does not fail + self.assertIn('test_regrtest_a', sys.modules) + self.assertIs(sys.modules['test_regrtest_b'], test_regrtest_b) + self.assertIs(sys.modules['test_regrtest_b.util'], test_regrtest_b.util) + self.assertNotIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py new file mode 100644 index 00000000000000..3dfba253455ad2 --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py @@ -0,0 +1,9 @@ +import sys +import unittest + +class Test(unittest.TestCase): + def test(self): + self.assertNotIn('test_regrtest_a', sys.modules) + self.assertIn('test_regrtest_b', sys.modules) + self.assertNotIn('test_regrtest_b.util', sys.modules) + self.assertNotIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/util.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/util.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py new file mode 100644 index 00000000000000..de80769118d709 --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py @@ -0,0 +1,11 @@ +import sys +import unittest +import test_regrtest_b.util + +class Test(unittest.TestCase): + def test(self): + test_regrtest_b.util # does not fail + self.assertNotIn('test_regrtest_a', sys.modules) + self.assertIs(sys.modules['test_regrtest_b'], test_regrtest_b) + self.assertIs(sys.modules['test_regrtest_b.util'], test_regrtest_b.util) + self.assertIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index eec5498e633eb6..e5fb725a30b5b8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -392,10 +392,10 @@ def skip_if_buildbot(reason=None): isbuildbot = False return unittest.skipIf(isbuildbot, reason) -def check_sanitizer(*, address=False, memory=False, ub=False): +def check_sanitizer(*, address=False, memory=False, ub=False, thread=False): """Returns True if Python is compiled with sanitizer support""" - if not (address or memory or ub): - raise ValueError('At least one of address, memory, or ub must be True') + if not (address or memory or ub or thread): + raise ValueError('At least one of address, memory, ub or thread must be True') cflags = sysconfig.get_config_var('CFLAGS') or '' @@ -412,18 +412,23 @@ def check_sanitizer(*, address=False, memory=False, ub=False): '-fsanitize=undefined' in cflags or '--with-undefined-behavior-sanitizer' in config_args ) + thread_sanitizer = ( + '-fsanitize=thread' in cflags or + '--with-thread-sanitizer' in config_args + ) return ( (memory and memory_sanitizer) or (address and address_sanitizer) or - (ub and ub_sanitizer) + (ub and ub_sanitizer) or + (thread and thread_sanitizer) ) -def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): +def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False, thread=False): """Decorator raising SkipTest if running with a sanitizer active.""" if not reason: reason = 'not working with sanitizers active' - skip = check_sanitizer(address=address, memory=memory, ub=ub) + skip = check_sanitizer(address=address, memory=memory, ub=ub, thread=thread) return unittest.skipIf(skip, reason) # gh-89363: True if fork() can hang if Python is built with Address Sanitizer @@ -432,7 +437,7 @@ def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): def set_sanitizer_env_var(env, option): - for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS'): + for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS', 'TSAN_OPTIONS'): if name in env: env[name] += f':{option}' else: @@ -796,7 +801,8 @@ def check_cflags_pgo(): return any(option in cflags_nodist for option in pgo_options) -if sysconfig.get_config_var('Py_GIL_DISABLED'): +Py_GIL_DISABLED = bool(sysconfig.get_config_var('Py_GIL_DISABLED')) +if Py_GIL_DISABLED: _header = 'PHBBInP' else: _header = 'nP' @@ -1843,7 +1849,7 @@ def restore(self): def with_pymalloc(): import _testcapi - return _testcapi.WITH_PYMALLOC + return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED def with_mimalloc(): @@ -2121,19 +2127,11 @@ def set_recursion_limit(limit): sys.setrecursionlimit(original_limit) def infinite_recursion(max_depth=None): - """Set a lower limit for tests that interact with infinite recursions - (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some - debug windows builds, due to not enough functions being inlined the - stack size might not handle the default recursion limit (1000). See - bpo-11105 for details.""" if max_depth is None: - if not python_is_optimized() or Py_DEBUG: - # Python built without compiler optimizations or in debug mode - # usually consumes more stack memory per function call. - # Unoptimized number based on what works under a WASI debug build. - max_depth = 50 - else: - max_depth = 100 + # Pick a number large enough to cause problems + # but not take too long for code that can handle + # very deep recursion. + max_depth = 20_000 elif max_depth < 3: raise ValueError("max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() @@ -2372,19 +2370,21 @@ def adjust_int_max_str_digits(max_digits): finally: sys.set_int_max_str_digits(current) -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = 5000 def _get_c_recursion_limit(): try: import _testcapi return _testcapi.Py_C_RECURSION_LIMIT except (ImportError, AttributeError): - return 1500 # (from Include/cpython/pystate.h) + # Originally taken from Include/cpython/pystate.h . + return 8000 # The default C recursion limit. Py_C_RECURSION_LIMIT = _get_c_recursion_limit() +#For recursion tests, easily exceeds default recursion limit +EXCEEDS_RECURSION_LIMIT = Py_C_RECURSION_LIMIT * 3 + #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', 'skipped on s390x') diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 388d1266773c8a..a4845065a5322e 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -7,6 +7,18 @@ _UNSPECIFIED = object() +def instructions_with_positions(instrs, co_positions): + # Return (instr, positions) pairs from the instrs list and co_positions + # iterator. The latter contains items for cache lines and the former + # doesn't, so those need to be skipped. + + co_positions = co_positions or iter(()) + for instr in instrs: + yield instr, next(co_positions, ()) + for _, size, _ in (instr.cache_info or ()): + for i in range(size): + next(co_positions, ()) + class BytecodeTestCase(unittest.TestCase): """Custom assertion methods for inspecting bytecode.""" diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py new file mode 100644 index 00000000000000..15a908e9663593 --- /dev/null +++ b/Lib/test/support/interpreters/__init__.py @@ -0,0 +1,187 @@ +"""Subinterpreters High Level Module.""" + +import threading +import weakref +import _xxsubinterpreters as _interpreters + +# aliases: +from _xxsubinterpreters import ( + InterpreterError, InterpreterNotFoundError, + is_shareable, +) + + +__all__ = [ + 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', + 'Interpreter', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', + 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', +] + + +_queuemod = None + +def __getattr__(name): + if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): + global create_queue, Queue, QueueEmpty, QueueFull + ns = globals() + from .queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, + ) + return ns[name] + else: + raise AttributeError(name) + + +_EXEC_FAILURE_STR = """ +{superstr} + +Uncaught in the interpreter: + +{formatted} +""".strip() + +class ExecFailure(RuntimeError): + + def __init__(self, excinfo): + msg = excinfo.formatted + if not msg: + if excinfo.type and excinfo.msg: + msg = f'{excinfo.type.__name__}: {excinfo.msg}' + else: + msg = excinfo.type.__name__ or excinfo.msg + super().__init__(msg) + self.excinfo = excinfo + + def __str__(self): + try: + formatted = self.excinfo.errdisplay + except Exception: + return super().__str__() + else: + return _EXEC_FAILURE_STR.format( + superstr=super().__str__(), + formatted=formatted, + ) + + +def create(): + """Return a new (idle) Python interpreter.""" + id = _interpreters.create(isolated=True) + return Interpreter(id) + + +def list_all(): + """Return all existing interpreters.""" + return [Interpreter(id) for id in _interpreters.list_all()] + + +def get_current(): + """Return the currently running interpreter.""" + id = _interpreters.get_current() + return Interpreter(id) + + +def get_main(): + """Return the main interpreter.""" + id = _interpreters.get_main() + return Interpreter(id) + + +_known = weakref.WeakValueDictionary() + +class Interpreter: + """A single Python interpreter.""" + + def __new__(cls, id, /): + # There is only one instance for any given ID. + if not isinstance(id, int): + raise TypeError(f'id must be an int, got {id!r}') + id = int(id) + try: + self = _known[id] + assert hasattr(self, '_ownsref') + except KeyError: + # This may raise InterpreterNotFoundError: + _interpreters._incref(id) + try: + self = super().__new__(cls) + self._id = id + self._ownsref = True + except BaseException: + _interpreters._deccref(id) + raise + _known[id] = self + return self + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + def __del__(self): + self._decref() + + def _decref(self): + if not self._ownsref: + return + self._ownsref = False + try: + _interpreters._decref(self.id) + except InterpreterNotFoundError: + pass + + @property + def id(self): + return self._id + + def is_running(self): + """Return whether or not the identified interpreter is running.""" + return _interpreters.is_running(self._id) + + def close(self): + """Finalize and destroy the interpreter. + + Attempting to destroy the current interpreter results + in a RuntimeError. + """ + return _interpreters.destroy(self._id) + + def prepare_main(self, ns=None, /, **kwargs): + """Bind the given values into the interpreter's __main__. + + The values must be shareable. + """ + ns = dict(ns, **kwargs) if ns is not None else kwargs + _interpreters.set___main___attrs(self._id, ns) + + def exec_sync(self, code, /): + """Run the given source code in the interpreter. + + This is essentially the same as calling the builtin "exec" + with this interpreter, using the __dict__ of its __main__ + module as both globals and locals. + + There is no return value. + + If the code raises an unhandled exception then an ExecFailure + is raised, which summarizes the unhandled exception. The actual + exception is discarded because objects cannot be shared between + interpreters. + + This blocks the current Python thread until done. During + that time, the previous interpreter is allowed to run + in other threads. + """ + excinfo = _interpreters.exec(self._id, code) + if excinfo is not None: + raise ExecFailure(excinfo) + + def run(self, code, /): + def task(): + self.exec_sync(code) + t = threading.Thread(target=task) + t.start() + return t diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters/channels.py similarity index 56% rename from Lib/test/support/interpreters.py rename to Lib/test/support/interpreters/channels.py index 089fe7ef56df78..75a5a60f54f926 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters/channels.py @@ -1,11 +1,9 @@ -"""Subinterpreters High Level Module.""" +"""Cross-interpreter Channels High Level Module.""" import time -import _xxsubinterpreters as _interpreters import _xxinterpchannels as _channels # aliases: -from _xxsubinterpreters import is_shareable from _xxinterpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, ChannelEmptyError, ChannelNotEmptyError, @@ -13,123 +11,13 @@ __all__ = [ - 'Interpreter', 'get_current', 'get_main', 'create', 'list_all', - 'RunFailedError', + 'create', 'list_all', 'SendChannel', 'RecvChannel', - 'create_channel', 'list_all_channels', 'is_shareable', - 'ChannelError', 'ChannelNotFoundError', - 'ChannelEmptyError', - ] + 'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError', +] -class RunFailedError(RuntimeError): - - def __init__(self, excinfo): - msg = excinfo.formatted - if not msg: - if excinfo.type and snapshot.msg: - msg = f'{snapshot.type.__name__}: {snapshot.msg}' - else: - msg = snapshot.type.__name__ or snapshot.msg - super().__init__(msg) - self.snapshot = excinfo - - -def create(*, isolated=True): - """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=isolated) - return Interpreter(id, isolated=isolated) - - -def list_all(): - """Return all existing interpreters.""" - return [Interpreter(id) for id in _interpreters.list_all()] - - -def get_current(): - """Return the currently running interpreter.""" - id = _interpreters.get_current() - return Interpreter(id) - - -def get_main(): - """Return the main interpreter.""" - id = _interpreters.get_main() - return Interpreter(id) - - -class Interpreter: - """A single Python interpreter.""" - - def __init__(self, id, *, isolated=None): - if not isinstance(id, (int, _interpreters.InterpreterID)): - raise TypeError(f'id must be an int, got {id!r}') - self._id = id - self._isolated = isolated - - def __repr__(self): - data = dict(id=int(self._id), isolated=self._isolated) - kwargs = (f'{k}={v!r}' for k, v in data.items()) - return f'{type(self).__name__}({", ".join(kwargs)})' - - def __hash__(self): - return hash(self._id) - - def __eq__(self, other): - if not isinstance(other, Interpreter): - return NotImplemented - else: - return other._id == self._id - - @property - def id(self): - return self._id - - @property - def isolated(self): - if self._isolated is None: - # XXX The low-level function has not been added yet. - # See bpo-.... - self._isolated = _interpreters.is_isolated(self._id) - return self._isolated - - def is_running(self): - """Return whether or not the identified interpreter is running.""" - return _interpreters.is_running(self._id) - - def close(self): - """Finalize and destroy the interpreter. - - Attempting to destroy the current interpreter results - in a RuntimeError. - """ - return _interpreters.destroy(self._id) - - # XXX Rename "run" to "exec"? - def run(self, src_str, /, channels=None): - """Run the given source code in the interpreter. - - This is essentially the same as calling the builtin "exec" - with this interpreter, using the __dict__ of its __main__ - module as both globals and locals. - - There is no return value. - - If the code raises an unhandled exception then a RunFailedError - is raised, which summarizes the unhandled exception. The actual - exception is discarded because objects cannot be shared between - interpreters. - - This blocks the current Python thread until done. During - that time, the previous interpreter is allowed to run - in other threads. - """ - excinfo = _interpreters.exec(self._id, src_str, channels) - if excinfo is not None: - raise RunFailedError(excinfo) - - -def create_channel(): +def create(): """Return (recv, send) for a new cross-interpreter channel. The channel may be used to pass data safely between interpreters. @@ -139,7 +27,7 @@ def create_channel(): return recv, send -def list_all_channels(): +def list_all(): """Return a list of (recv, send) for all open channels.""" return [(RecvChannel(cid), SendChannel(cid)) for cid in _channels.list_all()] diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py new file mode 100644 index 00000000000000..aead0c40ca9667 --- /dev/null +++ b/Lib/test/support/interpreters/queues.py @@ -0,0 +1,172 @@ +"""Cross-interpreter Queues High Level Module.""" + +import queue +import time +import weakref +import _xxinterpqueues as _queues + +# aliases: +from _xxinterpqueues import ( + QueueError, QueueNotFoundError, +) + +__all__ = [ + 'create', 'list_all', + 'Queue', + 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', +] + + +class QueueEmpty(_queues.QueueEmpty, queue.Empty): + """Raised from get_nowait() when the queue is empty. + + It is also raised from get() if it times out. + """ + + +class QueueFull(_queues.QueueFull, queue.Full): + """Raised from put_nowait() when the queue is full. + + It is also raised from put() if it times out. + """ + + +def create(maxsize=0): + """Return a new cross-interpreter queue. + + The queue may be used to pass data safely between interpreters. + """ + qid = _queues.create(maxsize) + return Queue(qid) + + +def list_all(): + """Return a list of all open queues.""" + return [Queue(qid) + for qid in _queues.list_all()] + + + +_known_queues = weakref.WeakValueDictionary() + +class Queue: + """A cross-interpreter queue.""" + + def __new__(cls, id, /): + # There is only one instance for any given ID. + if isinstance(id, int): + id = int(id) + else: + raise TypeError(f'id must be an int, got {id!r}') + try: + self = _known_queues[id] + except KeyError: + self = super().__new__(cls) + self._id = id + _known_queues[id] = self + _queues.bind(id) + return self + + def __del__(self): + try: + _queues.release(self._id) + except QueueNotFoundError: + pass + try: + del _known_queues[self._id] + except KeyError: + pass + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + @property + def id(self): + return self._id + + @property + def maxsize(self): + try: + return self._maxsize + except AttributeError: + self._maxsize = _queues.get_maxsize(self._id) + return self._maxsize + + def empty(self): + return self.qsize() == 0 + + def full(self): + return _queues.is_full(self._id) + + def qsize(self): + return _queues.get_count(self._id) + + def put(self, obj, timeout=None, *, + _delay=10 / 1000, # 10 milliseconds + ): + """Add the object to the queue. + + This blocks while the queue is full. + """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + while True: + try: + _queues.put(self._id, obj) + except _queues.QueueFull as exc: + if timeout is not None and time.time() >= end: + exc.__class__ = QueueFull + raise # re-raise + time.sleep(_delay) + else: + break + + def put_nowait(self, obj): + try: + return _queues.put(self._id, obj) + except _queues.QueueFull as exc: + exc.__class__ = QueueFull + raise # re-raise + + def get(self, timeout=None, *, + _delay=10 / 1000, # 10 milliseconds + ): + """Return the next object from the queue. + + This blocks while the queue is empty. + """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + while True: + try: + return _queues.get(self._id) + except _queues.QueueEmpty as exc: + if timeout is not None and time.time() >= end: + exc.__class__ = QueueEmpty + raise # re-raise + time.sleep(_delay) + return obj + + def get_nowait(self): + """Return the next object from the channel. + + If the queue is empty then raise QueueEmpty. Otherwise this + is the same as get(). + """ + try: + return _queues.get(self._id) + except _queues.QueueEmpty as exc: + exc.__class__ = QueueEmpty + raise # re-raise + + +_queues._register_queue_type(Queue) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 46ae53aa11a91f..20f38fd36a8876 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -247,15 +247,15 @@ def can_chmod(): global _can_chmod if _can_chmod is not None: return _can_chmod - if not hasattr(os, "chown"): + if not hasattr(os, "chmod"): _can_chmod = False return _can_chmod try: with open(TESTFN, "wb") as f: try: - os.chmod(TESTFN, 0o777) + os.chmod(TESTFN, 0o555) mode1 = os.stat(TESTFN).st_mode - os.chmod(TESTFN, 0o666) + os.chmod(TESTFN, 0o777) mode2 = os.stat(TESTFN).st_mode except OSError as e: can = False @@ -302,6 +302,10 @@ def can_dac_override(): else: _can_dac_override = True finally: + try: + os.chmod(TESTFN, 0o700) + except OSError: + pass unlink(TESTFN) return _can_dac_override @@ -592,10 +596,17 @@ def fd_count(): """Count the number of open file descriptors. """ if sys.platform.startswith(('linux', 'freebsd', 'emscripten')): + fd_path = "/proc/self/fd" + elif sys.platform == "darwin": + fd_path = "/dev/fd" + else: + fd_path = None + + if fd_path is not None: try: - names = os.listdir("/proc/self/fd") + names = os.listdir(fd_path) # Subtract one because listdir() internally opens a file - # descriptor to list the content of the /proc/self/fd/ directory. + # descriptor to list the content of the directory. return len(names) - 1 except FileNotFoundError: pass diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index 7dffe79a0da04e..759020c33aa700 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -92,13 +92,28 @@ def fail(self, cmd_line): # Executing the interpreter in a subprocess @support.requires_subprocess() def run_python_until_end(*args, **env_vars): + """Used to implement assert_python_*. + + *args are the command line flags to pass to the python interpreter. + **env_vars keyword arguments are environment variables to set on the process. + + If __run_using_command= is supplied, it must be a list of + command line arguments to prepend to the command line used. + Useful when you want to run another command that should launch the + python interpreter via its own arguments. ["/bin/echo", "--"] for + example could print the unquoted python command line instead of + run it. + """ env_required = interpreter_requires_environment() + run_using_command = env_vars.pop('__run_using_command', None) cwd = env_vars.pop('__cwd', None) if '__isolated' in env_vars: isolated = env_vars.pop('__isolated') else: isolated = not env_vars and not env_required cmd_line = [sys.executable, '-X', 'faulthandler'] + if run_using_command: + cmd_line = run_using_command + cmd_line if isolated: # isolated mode: ignore Python environment variables, ignore user # site-packages, and don't add the current directory to sys.path diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index 2b75e2f1916c82..cc2ed7849b0c0f 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -79,8 +79,7 @@ def __new__(cls, name=None, id=None): name = 'interp' elif name == 'main': raise ValueError('name mismatch (unexpected "main")') - if not isinstance(id, interpreters.InterpreterID): - id = interpreters.InterpreterID(id) + assert isinstance(id, int), repr(id) elif not name or name == 'main': name = 'main' id = main @@ -587,12 +586,12 @@ def test_run_string_arg_unresolved(self): cid = channels.create() interp = interpreters.create() + interpreters.set___main___attrs(interp, dict(cid=cid.send)) out = _run_output(interp, dedent(""" import _xxinterpchannels as _channels print(cid.end) _channels.send(cid, b'spam', blocking=False) - """), - dict(cid=cid.send)) + """)) obj = channels.recv(cid) self.assertEqual(obj, b'spam') diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 64a9db95e5eaf5..a76e4d0ade5b8a 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -15,6 +15,7 @@ interpreters = import_helper.import_module('_xxsubinterpreters') +from _xxsubinterpreters import InterpreterNotFoundError ################################## @@ -32,10 +33,10 @@ def _captured_script(script): return wrapped, open(r, encoding="utf-8") -def _run_output(interp, request, shared=None): +def _run_output(interp, request): script, rpipe = _captured_script(request) with rpipe: - interpreters.run_string(interp, script, shared) + interpreters.run_string(interp, script) return rpipe.read() @@ -266,7 +267,7 @@ def test_main(self): main = interpreters.get_main() cur = interpreters.get_current() self.assertEqual(cur, main) - self.assertIsInstance(cur, interpreters.InterpreterID) + self.assertIsInstance(cur, int) def test_subinterpreter(self): main = interpreters.get_main() @@ -275,7 +276,7 @@ def test_subinterpreter(self): import _xxsubinterpreters as _interpreters cur = _interpreters.get_current() print(cur) - assert isinstance(cur, _interpreters.InterpreterID) + assert isinstance(cur, int) """)) cur = int(out.strip()) _, expected = interpreters.list_all() @@ -289,7 +290,7 @@ def test_from_main(self): [expected] = interpreters.list_all() main = interpreters.get_main() self.assertEqual(main, expected) - self.assertIsInstance(main, interpreters.InterpreterID) + self.assertIsInstance(main, int) def test_from_subinterpreter(self): [expected] = interpreters.list_all() @@ -298,7 +299,7 @@ def test_from_subinterpreter(self): import _xxsubinterpreters as _interpreters main = _interpreters.get_main() print(main) - assert isinstance(main, _interpreters.InterpreterID) + assert isinstance(main, int) """)) main = int(out.strip()) self.assertEqual(main, expected) @@ -333,11 +334,11 @@ def test_from_subinterpreter(self): def test_already_destroyed(self): interp = interpreters.create() interpreters.destroy(interp) - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.is_running(interp) def test_does_not_exist(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.is_running(1_000_000) def test_bad_id(self): @@ -345,70 +346,11 @@ def test_bad_id(self): interpreters.is_running(-1) -class InterpreterIDTests(TestBase): - - def test_with_int(self): - id = interpreters.InterpreterID(10, force=True) - - self.assertEqual(int(id), 10) - - def test_coerce_id(self): - class Int(str): - def __index__(self): - return 10 - - id = interpreters.InterpreterID(Int(), force=True) - self.assertEqual(int(id), 10) - - def test_bad_id(self): - self.assertRaises(TypeError, interpreters.InterpreterID, object()) - self.assertRaises(TypeError, interpreters.InterpreterID, 10.0) - self.assertRaises(TypeError, interpreters.InterpreterID, '10') - self.assertRaises(TypeError, interpreters.InterpreterID, b'10') - self.assertRaises(ValueError, interpreters.InterpreterID, -1) - self.assertRaises(OverflowError, interpreters.InterpreterID, 2**64) - - def test_does_not_exist(self): - id = interpreters.create() - with self.assertRaises(RuntimeError): - interpreters.InterpreterID(int(id) + 1) # unforced - - def test_str(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(str(id), '10') - - def test_repr(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(repr(id), 'InterpreterID(10)') - - def test_equality(self): - id1 = interpreters.create() - id2 = interpreters.InterpreterID(int(id1)) - id3 = interpreters.create() - - self.assertTrue(id1 == id1) - self.assertTrue(id1 == id2) - self.assertTrue(id1 == int(id1)) - self.assertTrue(int(id1) == id1) - self.assertTrue(id1 == float(int(id1))) - self.assertTrue(float(int(id1)) == id1) - self.assertFalse(id1 == float(int(id1)) + 0.1) - self.assertFalse(id1 == str(int(id1))) - self.assertFalse(id1 == 2**1000) - self.assertFalse(id1 == float('inf')) - self.assertFalse(id1 == 'spam') - self.assertFalse(id1 == id3) - - self.assertFalse(id1 != id1) - self.assertFalse(id1 != id2) - self.assertTrue(id1 != id3) - - class CreateTests(TestBase): def test_in_main(self): id = interpreters.create() - self.assertIsInstance(id, interpreters.InterpreterID) + self.assertIsInstance(id, int) self.assertIn(id, interpreters.list_all()) @@ -444,7 +386,7 @@ def test_in_subinterpreter(self): import _xxsubinterpreters as _interpreters id = _interpreters.create() print(id) - assert isinstance(id, _interpreters.InterpreterID) + assert isinstance(id, int) """)) id2 = int(out.strip()) @@ -536,11 +478,11 @@ def f(): def test_already_destroyed(self): id = interpreters.create() interpreters.destroy(id) - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.destroy(id) def test_does_not_exist(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.destroy(1_000_000) def test_bad_id(self): @@ -688,10 +630,10 @@ def test_shareable_types(self): ] for obj in objects: with self.subTest(obj): + interpreters.set___main___attrs(interp, dict(obj=obj)) interpreters.run_string( interp, f'assert(obj == {obj!r})', - shared=dict(obj=obj), ) def test_os_exec(self): @@ -741,7 +683,7 @@ def test_does_not_exist(self): id = 0 while id in interpreters.list_all(): id += 1 - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.run_string(id, 'print("spam")') def test_error_id(self): @@ -779,7 +721,8 @@ def test_with_shared(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script, shared) + interpreters.set___main___attrs(self.id, shared) + interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -800,7 +743,8 @@ def test_shared_overwrites(self): ns2 = dict(vars()) del ns2['__builtins__'] """) - interpreters.run_string(self.id, script, shared) + interpreters.set___main___attrs(self.id, shared) + interpreters.run_string(self.id, script) r, w = os.pipe() script = dedent(f""" @@ -831,7 +775,8 @@ def test_shared_overwrites_default_vars(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script, shared) + interpreters.set___main___attrs(self.id, shared) + interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -1094,7 +1039,8 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') - interpreters.run_func(self.id, script, shared=dict(w=w)) + interpreters.set___main___attrs(self.id, dict(w=w)) + interpreters.run_func(self.id, script) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1110,7 +1056,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') def f(): - interpreters.run_func(self.id, script, shared=dict(w=w)) + interpreters.set___main___attrs(self.id, dict(w=w)) + interpreters.run_func(self.id, script) t = threading.Thread(target=f) t.start() t.join() @@ -1130,7 +1077,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') code = script.__code__ - interpreters.run_func(self.id, code, shared=dict(w=w)) + interpreters.set___main___attrs(self.id, dict(w=w)) + interpreters.run_func(self.id, code) with open(r, encoding="utf-8") as outfile: out = outfile.read() diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 13493d3c806d6a..266f057f0776c3 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -495,6 +495,24 @@ async def coro(): self.assertEqual(1, policy.set_event_loop.call_count) runner.close() + def test_no_repr_is_call_on_the_task_result(self): + # See https://github.com/python/cpython/issues/112559. + class MyResult: + def __init__(self): + self.repr_count = 0 + def __repr__(self): + self.repr_count += 1 + return super().__repr__() + + async def coro(): + return MyResult() + + + with asyncio.Runner() as runner: + result = runner.run(coro()) + + self.assertEqual(0, result.repr_count) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 37d015339761c6..f5f0afeab51c9e 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -47,6 +47,7 @@ def connection_made(self, ssl_proto, *, do_handshake=None): sslobj = mock.Mock() # emulate reading decompressed data sslobj.read.side_effect = ssl.SSLWantReadError + sslobj.write.side_effect = ssl.SSLWantReadError if do_handshake is not None: sslobj.do_handshake = do_handshake ssl_proto._sslobj = sslobj @@ -120,7 +121,19 @@ def test_close_during_handshake(self): test_utils.run_briefly(self.loop) ssl_proto._app_transport.close() - self.assertTrue(transport.abort.called) + self.assertTrue(transport._force_close.called) + + def test_close_during_ssl_over_ssl(self): + # gh-113214: passing exceptions from the inner wrapped SSL protocol to the + # shim transport provided by the outer SSL protocol should not raise + # attribute errors + outer = self.ssl_protocol(proto=self.ssl_protocol()) + self.connection_made(outer) + # Closing the outer app transport should not raise an exception + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + outer._app_transport.close() + self.assertEqual(messages, []) def test_get_extra_info_on_closed_connection(self): waiter = self.loop.create_future() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index b7966f8f03875b..e15492783aeec1 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -627,6 +627,11 @@ def __dir__(self): # test that object has a __dir__() self.assertEqual(sorted([].__dir__()), dir([])) + def test___ne__(self): + self.assertFalse(None.__ne__(None)) + self.assertTrue(None.__ne__(0)) + self.assertTrue(None.__ne__("abc")) + def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) self.assertEqual(divmod(-12, 7), (-2, 2)) @@ -832,6 +837,32 @@ class customdict(dict): # this one should not do anything fancy self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", exec, code, {'__builtins__': customdict()}) + def test_eval_builtins_mapping(self): + code = compile("superglobal", "test", "eval") + # works correctly + ns = {'__builtins__': types.MappingProxyType({'superglobal': 1})} + self.assertEqual(eval(code, ns), 1) + # custom builtins mapping is missing key + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", + eval, code, ns) + + def test_exec_builtins_mapping_import(self): + code = compile("import foo.bar", "test", "exec") + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(ImportError, "__import__ not found", exec, code, ns) + ns = {'__builtins__': types.MappingProxyType({'__import__': lambda *args: args})} + exec(code, ns) + self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0)) + + def test_eval_builtins_mapping_reduce(self): + # list_iterator.__reduce__() calls _PyEval_GetBuiltin("iter") + code = compile("x.__reduce__()", "test", "eval") + ns = {'__builtins__': types.MappingProxyType({}), 'x': iter([1, 2])} + self.assertRaisesRegex(AttributeError, "iter", eval, code, ns) + ns = {'__builtins__': types.MappingProxyType({'iter': iter}), 'x': iter([1, 2])} + self.assertEqual(eval(code, ns), (iter, ([1, 2],), 0)) + def test_exec_redirected(self): savestdout = sys.stdout sys.stdout = None # Whatever that cannot flush() @@ -2039,6 +2070,23 @@ def test_bytearray_extend_error(self): bad_iter = map(int, "X") self.assertRaises(ValueError, array.extend, bad_iter) + def test_bytearray_join_with_misbehaving_iterator(self): + # Issue #112625 + array = bytearray(b',') + def iterator(): + array.clear() + yield b'A' + yield b'B' + self.assertRaises(BufferError, array.join, iterator()) + + def test_bytearray_join_with_custom_iterator(self): + # Issue #112625 + array = bytearray(b',') + def iterator(): + yield b'A' + yield b'B' + self.assertEqual(bytearray(b'A,B'), array.join(iterator())) + def test_construct_singletons(self): for const in None, Ellipsis, NotImplemented: tp = type(const) @@ -2252,7 +2300,10 @@ def _run_child(self, child, terminal_input): return lines - def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): + def check_input_tty(self, prompt, terminal_input, stdio_encoding=None, *, + expected=None, + stdin_errors='surrogateescape', + stdout_errors='replace'): if not sys.stdin.isatty() or not sys.stdout.isatty(): self.skipTest("stdin and stdout must be ttys") def child(wpipe): @@ -2260,22 +2311,26 @@ def child(wpipe): if stdio_encoding: sys.stdin = io.TextIOWrapper(sys.stdin.detach(), encoding=stdio_encoding, - errors='surrogateescape') + errors=stdin_errors) sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding=stdio_encoding, - errors='replace') + errors=stdout_errors) print("tty =", sys.stdin.isatty() and sys.stdout.isatty(), file=wpipe) - print(ascii(input(prompt)), file=wpipe) + try: + print(ascii(input(prompt)), file=wpipe) + except BaseException as e: + print(ascii(f'{e.__class__.__name__}: {e!s}'), file=wpipe) lines = self.run_child(child, terminal_input + b"\r\n") # Check we did exercise the GNU readline path self.assertIn(lines[0], {'tty = True', 'tty = False'}) if lines[0] != 'tty = True': self.skipTest("standard IO in should have been a tty") input_result = eval(lines[1]) # ascii() -> eval() roundtrip - if stdio_encoding: - expected = terminal_input.decode(stdio_encoding, 'surrogateescape') - else: - expected = terminal_input.decode(sys.stdin.encoding) # what else? + if expected is None: + if stdio_encoding: + expected = terminal_input.decode(stdio_encoding, 'surrogateescape') + else: + expected = terminal_input.decode(sys.stdin.encoding) # what else? self.assertEqual(input_result, expected) def test_input_tty(self): @@ -2296,13 +2351,32 @@ def skip_if_readline(self): def test_input_tty_non_ascii(self): self.skip_if_readline() # Check stdin/stdout encoding is used when invoking PyOS_Readline() - self.check_input_tty("prompté", b"quux\xe9", "utf-8") + self.check_input_tty("prompté", b"quux\xc3\xa9", "utf-8") def test_input_tty_non_ascii_unicode_errors(self): self.skip_if_readline() # Check stdin/stdout error handler is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") + def test_input_tty_null_in_prompt(self): + self.check_input_tty("prompt\0", b"", + expected='ValueError: input: prompt string cannot contain ' + 'null characters') + + def test_input_tty_nonencodable_prompt(self): + self.skip_if_readline() + self.check_input_tty("prompté", b"quux", "ascii", stdout_errors='strict', + expected="UnicodeEncodeError: 'ascii' codec can't encode " + "character '\\xe9' in position 6: ordinal not in " + "range(128)") + + def test_input_tty_nondecodable_input(self): + self.skip_if_readline() + self.check_input_tty("prompt", b"quux\xe9", "ascii", stdin_errors='strict', + expected="UnicodeDecodeError: 'ascii' codec can't decode " + "byte 0xe9 in position 4: ordinal not in " + "range(128)") + def test_input_no_stdout_fileno(self): # Issue #24402: If stdin is the original terminal but stdout.fileno() # fails, do not use the original stdout file descriptor diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index afd506f07520d8..a3804a945f2e3a 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -46,6 +46,10 @@ def __index__(self): class BaseBytesTest: + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + def test_basics(self): b = self.type2test() self.assertEqual(type(b), self.type2test) @@ -1023,36 +1027,63 @@ def test_buffer_is_readonly(self): self.assertRaises(TypeError, f.readinto, b"") def test_custom(self): - class A: - def __bytes__(self): - return b'abc' - self.assertEqual(bytes(A()), b'abc') - class A: pass - self.assertRaises(TypeError, bytes, A()) - class A: - def __bytes__(self): - return None - self.assertRaises(TypeError, bytes, A()) - class A: + self.assertEqual(bytes(BytesSubclass(b'abc')), b'abc') + self.assertEqual(BytesSubclass(OtherBytesSubclass(b'abc')), + BytesSubclass(b'abc')) + self.assertEqual(bytes(WithBytes(b'abc')), b'abc') + self.assertEqual(BytesSubclass(WithBytes(b'abc')), BytesSubclass(b'abc')) + + class NoBytes: pass + self.assertRaises(TypeError, bytes, NoBytes()) + self.assertRaises(TypeError, bytes, WithBytes('abc')) + self.assertRaises(TypeError, bytes, WithBytes(None)) + class IndexWithBytes: def __bytes__(self): return b'a' def __index__(self): return 42 - self.assertEqual(bytes(A()), b'a') + self.assertEqual(bytes(IndexWithBytes()), b'a') # Issue #25766 - class A(str): + class StrWithBytes(str): + def __new__(cls, value): + self = str.__new__(cls, '\u20ac') + self.value = value + return self def __bytes__(self): - return b'abc' - self.assertEqual(bytes(A('\u20ac')), b'abc') - self.assertEqual(bytes(A('\u20ac'), 'iso8859-15'), b'\xa4') + return self.value + self.assertEqual(bytes(StrWithBytes(b'abc')), b'abc') + self.assertEqual(bytes(StrWithBytes(b'abc'), 'iso8859-15'), b'\xa4') + self.assertEqual(bytes(StrWithBytes(BytesSubclass(b'abc'))), b'abc') + self.assertEqual(BytesSubclass(StrWithBytes(b'abc')), BytesSubclass(b'abc')) + self.assertEqual(BytesSubclass(StrWithBytes(b'abc'), 'iso8859-15'), + BytesSubclass(b'\xa4')) + self.assertEqual(BytesSubclass(StrWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertEqual(BytesSubclass(StrWithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) # Issue #24731 - class A: + self.assertTypedEqual(bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(WithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(WithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + + class BytesWithBytes(bytes): + def __new__(cls, value): + self = bytes.__new__(cls, b'\xa4') + self.value = value + return self def __bytes__(self): - return OtherBytesSubclass(b'abc') - self.assertEqual(bytes(A()), b'abc') - self.assertIs(type(bytes(A())), OtherBytesSubclass) - self.assertEqual(BytesSubclass(A()), b'abc') - self.assertIs(type(BytesSubclass(A())), BytesSubclass) + return self.value + self.assertTypedEqual(bytes(BytesWithBytes(b'abc')), b'abc') + self.assertTypedEqual(BytesSubclass(BytesWithBytes(b'abc')), + BytesSubclass(b'abc')) + self.assertTypedEqual(bytes(BytesWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(BytesWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(BytesWithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) # Test PyBytes_FromFormat() def test_from_format(self): @@ -2069,6 +2100,12 @@ class BytesSubclass(bytes): class OtherBytesSubclass(bytes): pass +class WithBytes: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + class ByteArraySubclassTest(SubclassTest, unittest.TestCase): basetype = bytearray type2test = ByteArraySubclass diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 71f934756e26a1..7334a325ba22f0 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -112,12 +112,16 @@ class EncodingDetails(_EncodingDetails): ]) @classmethod - def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, env_vars): + def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, stream_errors, env_vars): """Returns expected child process details for a given encoding""" _stream = stream_encoding + ":{}" - # stdin and stdout should use surrogateescape either because the - # coercion triggered, or because the C locale was detected - stream_info = 2*[_stream.format("surrogateescape")] + if stream_errors is None: + # stdin and stdout should use surrogateescape either because the + # coercion triggered, or because the C locale was detected + stream_errors = "surrogateescape" + + stream_info = [_stream.format(stream_errors)] * 2 + # stderr should always use backslashreplace stream_info.append(_stream.format("backslashreplace")) expected_lang = env_vars.get("LANG", "not set") @@ -210,6 +214,7 @@ def _check_child_encoding_details(self, env_vars, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors, expected_warnings, coercion_expected): """Check the C locale handling for the given process environment @@ -225,6 +230,7 @@ def _check_child_encoding_details(self, coercion_expected, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors, env_vars ) self.assertEqual(encoding_details, expected_details) @@ -257,6 +263,7 @@ def test_external_target_locale_configuration(self): "LC_CTYPE": "", "LC_ALL": "", "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "", } for env_var in ("LANG", "LC_CTYPE"): for locale_to_set in AVAILABLE_TARGETS: @@ -273,10 +280,43 @@ def test_external_target_locale_configuration(self): self._check_child_encoding_details(var_dict, expected_fs_encoding, expected_stream_encoding, + expected_stream_errors=None, expected_warnings=None, coercion_expected=False) + def test_with_ioencoding(self): + # Explicitly setting a target locale should give the same behaviour as + # is seen when implicitly coercing to that target locale + self.maxDiff = None + + expected_fs_encoding = "utf-8" + expected_stream_encoding = "utf-8" + base_var_dict = { + "LANG": "", + "LC_CTYPE": "", + "LC_ALL": "", + "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "UTF-8", + } + for env_var in ("LANG", "LC_CTYPE"): + for locale_to_set in AVAILABLE_TARGETS: + # XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as + # expected, so skip that combination for now + # See https://bugs.python.org/issue30672 for discussion + if env_var == "LANG" and locale_to_set == "UTF-8": + continue + + with self.subTest(env_var=env_var, + configured_locale=locale_to_set): + var_dict = base_var_dict.copy() + var_dict[env_var] = locale_to_set + self._check_child_encoding_details(var_dict, + expected_fs_encoding, + expected_stream_encoding, + expected_stream_errors="strict", + expected_warnings=None, + coercion_expected=False) @support.cpython_only @unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"), @@ -316,6 +356,7 @@ def _check_c_locale_coercion(self, "LC_CTYPE": "", "LC_ALL": "", "PYTHONCOERCECLOCALE": "", + "PYTHONIOENCODING": "", } base_var_dict.update(extra_vars) if coerce_c_locale is not None: @@ -340,6 +381,7 @@ def _check_c_locale_coercion(self, self._check_child_encoding_details(base_var_dict, fs_encoding, stream_encoding, + None, _expected_warnings, _coercion_expected) @@ -348,13 +390,15 @@ def _check_c_locale_coercion(self, for env_var in ("LANG", "LC_CTYPE"): with self.subTest(env_var=env_var, nominal_locale=locale_to_set, - PYTHONCOERCECLOCALE=coerce_c_locale): + PYTHONCOERCECLOCALE=coerce_c_locale, + PYTHONIOENCODING=""): var_dict = base_var_dict.copy() var_dict[env_var] = locale_to_set # Check behaviour on successful coercion self._check_child_encoding_details(var_dict, fs_encoding, stream_encoding, + None, expected_warnings, coercion_expected) diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index 26152c3049848c..97ed939928c360 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -8,6 +8,30 @@ NULL = None +class StrSubclass(str): + pass + +class BytesSubclass(bytes): + pass + +class WithStr: + def __init__(self, value): + self.value = value + def __str__(self): + return self.value + +class WithRepr: + def __init__(self, value): + self.value = value + def __repr__(self): + return self.value + +class WithBytes: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + class TestObject: @property def evil(self): @@ -44,6 +68,68 @@ def gen(): class CAPITest(unittest.TestCase): + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + + def test_object_str(self): + # Test PyObject_Str() + object_str = _testcapi.object_str + self.assertTypedEqual(object_str(''), '') + self.assertTypedEqual(object_str('abc'), 'abc') + self.assertTypedEqual(object_str('\U0001f40d'), '\U0001f40d') + self.assertTypedEqual(object_str(StrSubclass('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(object_str(WithRepr('')), '') + self.assertTypedEqual(object_str(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(object_str(NULL), '') + + def test_object_repr(self): + # Test PyObject_Repr() + object_repr = _testcapi.object_repr + self.assertTypedEqual(object_repr(''), "''") + self.assertTypedEqual(object_repr('abc'), "'abc'") + self.assertTypedEqual(object_repr('\U0001f40d'), "'\U0001f40d'") + self.assertTypedEqual(object_repr(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_repr(WithRepr('')), '') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(object_repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) + self.assertTypedEqual(object_repr(NULL), '') + + def test_object_ascii(self): + # Test PyObject_ASCII() + object_ascii = _testcapi.object_ascii + self.assertTypedEqual(object_ascii(''), "''") + self.assertTypedEqual(object_ascii('abc'), "'abc'") + self.assertTypedEqual(object_ascii('\U0001f40d'), r"'\U0001f40d'") + self.assertTypedEqual(object_ascii(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_ascii(WithRepr('')), '') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(object_ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(NULL), '') + + def test_object_bytes(self): + # Test PyObject_Bytes() + object_bytes = _testcapi.object_bytes + self.assertTypedEqual(object_bytes(b''), b'') + self.assertTypedEqual(object_bytes(b'abc'), b'abc') + self.assertTypedEqual(object_bytes(BytesSubclass(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) + self.assertTypedEqual(object_bytes(bytearray(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(memoryview(b'abc')), b'abc') + self.assertTypedEqual(object_bytes([97, 98, 99]), b'abc') + self.assertTypedEqual(object_bytes((97, 98, 99)), b'abc') + self.assertTypedEqual(object_bytes(iter([97, 98, 99])), b'abc') + self.assertRaises(TypeError, object_bytes, WithBytes(bytearray(b'abc'))) + self.assertRaises(TypeError, object_bytes, WithBytes([97, 98, 99])) + self.assertRaises(TypeError, object_bytes, 3) + self.assertRaises(TypeError, object_bytes, 'abc') + self.assertRaises(TypeError, object_bytes, object()) + self.assertTypedEqual(object_bytes(NULL), b'') def test_object_getattr(self): xgetattr = _testcapi.object_getattr diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py index 9f51efb091dea5..d6fc1f077c40aa 100644 --- a/Lib/test/test_capi/test_complex.py +++ b/Lib/test/test_capi/test_complex.py @@ -1,3 +1,5 @@ +from math import isnan +import errno import unittest import warnings @@ -10,6 +12,10 @@ _testcapi = import_helper.import_module('_testcapi') NULL = None +INF = float("inf") +NAN = float("nan") +DBL_MAX = _testcapi.DBL_MAX + class BadComplex3: def __complex__(self): @@ -141,6 +147,87 @@ def test_asccomplex(self): # CRASHES asccomplex(NULL) + def test_py_c_sum(self): + # Test _Py_c_sum() + _py_c_sum = _testcapi._py_c_sum + + self.assertEqual(_py_c_sum(1, 1j), (1+1j, 0)) + + def test_py_c_diff(self): + # Test _Py_c_diff() + _py_c_diff = _testcapi._py_c_diff + + self.assertEqual(_py_c_diff(1, 1j), (1-1j, 0)) + + def test_py_c_neg(self): + # Test _Py_c_neg() + _py_c_neg = _testcapi._py_c_neg + + self.assertEqual(_py_c_neg(1+1j), -1-1j) + + def test_py_c_prod(self): + # Test _Py_c_prod() + _py_c_prod = _testcapi._py_c_prod + + self.assertEqual(_py_c_prod(2, 1j), (2j, 0)) + + def test_py_c_quot(self): + # Test _Py_c_quot() + _py_c_quot = _testcapi._py_c_quot + + self.assertEqual(_py_c_quot(1, 1j), (-1j, 0)) + self.assertEqual(_py_c_quot(1, -1j), (1j, 0)) + self.assertEqual(_py_c_quot(1j, 2), (0.5j, 0)) + self.assertEqual(_py_c_quot(1j, -2), (-0.5j, 0)) + self.assertEqual(_py_c_quot(1, 2j), (-0.5j, 0)) + + z, e = _py_c_quot(NAN, 1j) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + self.assertEqual(e, 0) + + z, e = _py_c_quot(1j, NAN) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + self.assertEqual(e, 0) + + self.assertEqual(_py_c_quot(1, 0j)[1], errno.EDOM) + + def test_py_c_pow(self): + # Test _Py_c_pow() + _py_c_pow = _testcapi._py_c_pow + + self.assertEqual(_py_c_pow(1j, 0j), (1+0j, 0)) + self.assertEqual(_py_c_pow(1, 1j), (1+0j, 0)) + self.assertEqual(_py_c_pow(0j, 1), (0j, 0)) + self.assertAlmostEqual(_py_c_pow(1j, 2)[0], -1.0+0j) + + r, e = _py_c_pow(1+1j, -1) + self.assertAlmostEqual(r, 0.5-0.5j) + self.assertEqual(e, 0) + + self.assertEqual(_py_c_pow(0j, -1)[1], errno.EDOM) + self.assertEqual(_py_c_pow(0j, 1j)[1], errno.EDOM) + self.assertEqual(_py_c_pow(*[DBL_MAX+1j]*2)[0], complex(*[INF]*2)) + + + def test_py_c_abs(self): + # Test _Py_c_abs() + _py_c_abs = _testcapi._py_c_abs + + self.assertEqual(_py_c_abs(-1), (1.0, 0)) + self.assertEqual(_py_c_abs(1j), (1.0, 0)) + + self.assertEqual(_py_c_abs(complex('+inf+1j')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('-inf+1j')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('1.25+infj')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('1.25-infj')), (INF, 0)) + + self.assertTrue(isnan(_py_c_abs(complex('1.25+nanj'))[0])) + self.assertTrue(isnan(_py_c_abs(complex('nan-1j'))[0])) + + self.assertEqual(_py_c_abs(complex(*[DBL_MAX]*2))[1], errno.ERANGE) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index c964b1efd577ba..9b6aef27625ad0 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1314,6 +1314,34 @@ def test_nonascii_keywords(self): f"'{name2}' is an invalid keyword argument"): parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) + def test_nested_tuple(self): + parse = _testcapi.parse_tuple_and_keywords + + self.assertEqual(parse(((1, 2, 3),), {}, '(OOO)', ['a']), (1, 2, 3)) + self.assertEqual(parse((1, (2, 3), 4), {}, 'O(OO)O', ['a', 'b', 'c']), + (1, 2, 3, 4)) + parse(((1, 2, 3),), {}, '(iii)', ['a']) + + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 2, not 3"): + parse(((1, 2, 3),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 2, not 1"): + parse(((1,),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item sequence, not int"): + parse((1,), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item sequence, not bytes"): + parse((b'ab',), {}, '(ii)', ['a']) + + for f in 'es', 'et', 'es#', 'et#': + with self.assertRaises(LookupError): # empty encoding "" + parse((('a',),), {}, '(' + f + ')', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 1, not 0"): + parse(((),), {}, '(' + f + ')', ['a']) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_hash.py b/Lib/test/test_capi/test_hash.py index 59dec15bc21445..8436da7c32df10 100644 --- a/Lib/test/test_capi/test_hash.py +++ b/Lib/test/test_capi/test_hash.py @@ -4,7 +4,8 @@ _testcapi = import_helper.import_module('_testcapi') -SIZEOF_PY_HASH_T = _testcapi.SIZEOF_VOID_P +SIZEOF_VOID_P = _testcapi.SIZEOF_VOID_P +SIZEOF_PY_HASH_T = SIZEOF_VOID_P class CAPITest(unittest.TestCase): @@ -31,3 +32,48 @@ def test_hash_getfuncdef(self): self.assertEqual(func_def.name, hash_info.algorithm) self.assertEqual(func_def.hash_bits, hash_info.hash_bits) self.assertEqual(func_def.seed_bits, hash_info.seed_bits) + + def test_hash_pointer(self): + # Test Py_HashPointer() + hash_pointer = _testcapi.hash_pointer + + UHASH_T_MASK = ((2 ** (8 * SIZEOF_PY_HASH_T)) - 1) + HASH_T_MAX = (2 ** (8 * SIZEOF_PY_HASH_T - 1) - 1) + + def python_hash_pointer(x): + # Py_HashPointer() rotates the pointer bits by 4 bits to the right + x = (x >> 4) | ((x & 15) << (8 * SIZEOF_VOID_P - 4)) + + # Convert unsigned uintptr_t (Py_uhash_t) to signed Py_hash_t + if HASH_T_MAX < x: + x = (~x) + 1 + x &= UHASH_T_MASK + x = (~x) + 1 + return x + + if SIZEOF_VOID_P == 8: + values = ( + 0xABCDEF1234567890, + 0x1234567890ABCDEF, + 0xFEE4ABEDD1CECA5E, + ) + else: + values = ( + 0x12345678, + 0x1234ABCD, + 0xDEADCAFE, + ) + + for value in values: + expected = python_hash_pointer(value) + with self.subTest(value=value): + self.assertEqual(hash_pointer(value), expected, + f"hash_pointer({value:x}) = " + f"{hash_pointer(value):x} != {expected:x}") + + # Py_HashPointer(NULL) returns 0 + self.assertEqual(hash_pointer(0), 0) + + # Py_HashPointer((void*)(uintptr_t)-1) doesn't return -1 but -2 + VOID_P_MAX = -1 & (2 ** (8 * SIZEOF_VOID_P) - 1) + self.assertEqual(hash_pointer(VOID_P_MAX), -2) diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 72f23b1a34080e..04f17a9ec9e72a 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -152,6 +152,8 @@ class C(): pass self.assertGreaterEqual(count, i*5-2) +# Py_GIL_DISABLED requires mimalloc (not malloc) +@unittest.skipIf(support.Py_GIL_DISABLED, 'need malloc') class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' @@ -161,6 +163,11 @@ class PyMemPymallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'pymalloc_debug' +@unittest.skipUnless(support.with_mimalloc(), 'need mimaloc') +class PyMemMimallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'mimalloc_debug' + + @unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') class PyMemDefaultTests(PyMemDebugTests): # test default allocator of Python compiled in debug mode diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 6cbf5d22203804..67fbef4f269814 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -7,7 +7,6 @@ import importlib.machinery import importlib.util import json -import opcode import os import pickle import queue @@ -470,6 +469,8 @@ def __del__(self): del L self.assertEqual(PyList.num, 0) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_heap_ctype_doc_and_text_signature(self): self.assertEqual(_testcapi.HeapDocCType.__doc__, "somedoc") self.assertEqual(_testcapi.HeapDocCType.__text_signature__, "(arg1, arg2)") @@ -1527,6 +1528,7 @@ def test_isolated_subinterpreter(self): maxtext = 250 main_interpid = 0 interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) _interpreters.run_string(interpid, f"""if True: import json import os @@ -2020,6 +2022,137 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +@requires_subinterpreters +class InterpreterIDTests(unittest.TestCase): + + InterpreterID = _testcapi.get_interpreterid_type() + + def new_interpreter(self): + def ensure_destroyed(interpid): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + id = _interpreters.create() + self.addCleanup(lambda: ensure_destroyed(id)) + return id + + def test_with_int(self): + id = self.InterpreterID(10, force=True) + + self.assertEqual(int(id), 10) + + def test_coerce_id(self): + class Int(str): + def __index__(self): + return 10 + + id = self.InterpreterID(Int(), force=True) + self.assertEqual(int(id), 10) + + def test_bad_id(self): + for badid in [ + object(), + 10.0, + '10', + b'10', + ]: + with self.subTest(badid): + with self.assertRaises(TypeError): + self.InterpreterID(badid) + + badid = -1 + with self.subTest(badid): + with self.assertRaises(ValueError): + self.InterpreterID(badid) + + badid = 2**64 + with self.subTest(badid): + with self.assertRaises(OverflowError): + self.InterpreterID(badid) + + def test_exists(self): + id = self.new_interpreter() + with self.assertRaises(_interpreters.InterpreterNotFoundError): + self.InterpreterID(int(id) + 1) # unforced + + def test_does_not_exist(self): + id = self.new_interpreter() + with self.assertRaises(_interpreters.InterpreterNotFoundError): + self.InterpreterID(int(id) + 1) # unforced + + def test_destroyed(self): + id = _interpreters.create() + _interpreters.destroy(id) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + self.InterpreterID(id) # unforced + + def test_str(self): + id = self.InterpreterID(10, force=True) + self.assertEqual(str(id), '10') + + def test_repr(self): + id = self.InterpreterID(10, force=True) + self.assertEqual(repr(id), 'InterpreterID(10)') + + def test_equality(self): + id1 = self.new_interpreter() + id2 = self.InterpreterID(id1) + id3 = self.InterpreterID( + self.new_interpreter()) + + self.assertTrue(id2 == id2) # identity + self.assertTrue(id2 == id1) # int-equivalent + self.assertTrue(id1 == id2) # reversed + self.assertTrue(id2 == int(id2)) + self.assertTrue(id2 == float(int(id2))) + self.assertTrue(float(int(id2)) == id2) + self.assertFalse(id2 == float(int(id2)) + 0.1) + self.assertFalse(id2 == str(int(id2))) + self.assertFalse(id2 == 2**1000) + self.assertFalse(id2 == float('inf')) + self.assertFalse(id2 == 'spam') + self.assertFalse(id2 == id3) + + self.assertFalse(id2 != id2) + self.assertFalse(id2 != id1) + self.assertFalse(id1 != id2) + self.assertTrue(id2 != id3) + + def test_linked_lifecycle(self): + id1 = _interpreters.create() + _testcapi.unlink_interpreter_refcount(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 0) + + id2 = self.InterpreterID(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 1) + + # The interpreter isn't linked to ID objects, so it isn't destroyed. + del id2 + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 0) + + _testcapi.link_interpreter_refcount(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 0) + + id3 = self.InterpreterID(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 1) + + # The interpreter is linked now so is destroyed. + del id3 + with self.assertRaises(_interpreters.InterpreterNotFoundError): + _testinternalcapi.get_interpreter_refcount(id1) + + class BuiltinStaticTypesTests(unittest.TestCase): TYPES = [ @@ -2351,507 +2484,59 @@ def func(): self.do_test(func, names) -@contextlib.contextmanager -def temporary_optimizer(opt): - old_opt = _testinternalcapi.get_optimizer() - _testinternalcapi.set_optimizer(opt) - try: - yield - finally: - _testinternalcapi.set_optimizer(old_opt) - - -@contextlib.contextmanager -def clear_executors(func): - # Clear executors in func before and after running a block - func.__code__ = func.__code__.replace() - try: - yield - finally: - func.__code__ = func.__code__.replace() - - -class TestOptimizerAPI(unittest.TestCase): - - def test_get_counter_optimizer_dealloc(self): - # See gh-108727 - def f(): - _testinternalcapi.get_counter_optimizer() +@unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED') +class TestPyThreadId(unittest.TestCase): + def test_py_thread_id(self): + # gh-112535: Test _Py_ThreadId(): make sure that thread identifiers + # in a few threads are unique + py_thread_id = _testinternalcapi.py_thread_id + short_sleep = 0.010 - f() - - def test_get_set_optimizer(self): - old = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() - try: - _testinternalcapi.set_optimizer(opt) - self.assertEqual(_testinternalcapi.get_optimizer(), opt) - _testinternalcapi.set_optimizer(None) - self.assertEqual(_testinternalcapi.get_optimizer(), None) - finally: - _testinternalcapi.set_optimizer(old) - - - def test_counter_optimizer(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def loop(): - for _ in range(1000): - pass - """), ns, ns) - loop = ns['loop'] - - for repeat in range(5): - opt = _testinternalcapi.get_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - with clear_executors(loop): - loop() - self.assertEqual(opt.get_count(), 1000) - - def test_long_loop(self): - "Check that we aren't confused by EXTENDED_ARG" - - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def nop(): - pass - - def long_loop(): - for _ in range(10): - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - """), ns, ns) - long_loop = ns['long_loop'] - - opt = _testinternalcapi.get_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - long_loop() - self.assertEqual(opt.get_count(), 10) - - def test_code_restore_for_ENTER_EXECUTOR(self): - def testfunc(x): - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.get_counter_optimizer() - with temporary_optimizer(opt): - testfunc(1000) - code, replace_code = testfunc.__code__, testfunc.__code__.replace() - self.assertEqual(code, replace_code) - self.assertEqual(hash(code), hash(replace_code)) - - -def get_first_executor(func): - code = func.__code__ - co_code = code.co_code - JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] - for i in range(0, len(co_code), 2): - if co_code[i] == JUMP_BACKWARD: - try: - return _testinternalcapi.get_executor(code, i) - except ValueError: - pass - return None - - -class TestExecutorInvalidation(unittest.TestCase): - - def setUp(self): - self.old = _testinternalcapi.get_optimizer() - self.opt = _testinternalcapi.get_counter_optimizer() - _testinternalcapi.set_optimizer(self.opt) - - def tearDown(self): - _testinternalcapi.set_optimizer(self.old) - - def test_invalidate_object(self): - # Generate a new set of functions at each call - ns = {} - func_src = "\n".join( - f""" - def f{n}(): - for _ in range(1000): - pass - """ for n in range(5) - ) - exec(textwrap.dedent(func_src), ns, ns) - funcs = [ ns[f'f{n}'] for n in range(5)] - objects = [object() for _ in range(5)] - - for f in funcs: - f() - executors = [get_first_executor(f) for f in funcs] - # Set things up so each executor depends on the objects - # with an equal or lower index. - for i, exe in enumerate(executors): - self.assertTrue(exe.is_valid()) - for obj in objects[:i+1]: - _testinternalcapi.add_executor_dependency(exe, obj) - self.assertTrue(exe.is_valid()) - # Assert that the correct executors are invalidated - # and check that nothing crashes when we invalidate - # an executor mutliple times. - for i in (4,3,2,1,0): - _testinternalcapi.invalidate_executors(objects[i]) - for exe in executors[i:]: - self.assertFalse(exe.is_valid()) - for exe in executors[:i]: - self.assertTrue(exe.is_valid()) - - def test_uop_optimizer_invalidation(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def f(): - for i in range(1000): - pass - """), ns, ns) - f = ns['f'] - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - f() - exe = get_first_executor(f) - self.assertTrue(exe.is_valid()) - _testinternalcapi.invalidate_executors(f.__code__) - self.assertFalse(exe.is_valid()) - -class TestUops(unittest.TestCase): - - def test_basic_loop(self): - def testfunc(x): - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(1000) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_SET_IP", uops) - self.assertIn("LOAD_FAST", uops) - - def test_extended_arg(self): - "Check EXTENDED_ARG handling in superblock creation" - ns = {} - exec(textwrap.dedent(""" - def many_vars(): - # 260 vars, so z9 should have index 259 - a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 - b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 - c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 - d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 - e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 - f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 - g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 - h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 - i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 - j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 - k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 - l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 - m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 - n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 - o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 - p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 - q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 - r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 - s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 - t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 - u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 - v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 - w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 - x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 - y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 - z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 - while z9 > 0: - z9 = z9 - 1 - """), ns, ns) - many_vars = ns["many_vars"] - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - ex = get_first_executor(many_vars) - self.assertIsNone(ex) - many_vars() - - ex = get_first_executor(many_vars) - self.assertIsNotNone(ex) - self.assertIn(("LOAD_FAST", 259, 0), list(ex)) - - def test_unspecialized_unpack(self): - # An example of an unspecialized opcode - def testfunc(x): - i = 0 - while i < x: - i += 1 - a, b = {1: 2, 3: 3} - assert a == 1 and b == 3 - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_UNPACK_SEQUENCE", uops) - - def test_pop_jump_if_false(self): - def testfunc(n): - i = 0 - while i < n: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_TRUE_POP", uops) - - def test_pop_jump_if_none(self): - def testfunc(a): - for x in a: - if x is None: - x = 0 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(range(20)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) - - def test_pop_jump_if_not_none(self): - def testfunc(a): - for x in a: - x = None - if x is not None: - x = 0 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(range(20)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_NONE_POP", uops) - - def test_pop_jump_if_true(self): - def testfunc(n): - i = 0 - while not i >= n: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_FALSE_POP", uops) - - def test_jump_backward(self): - def testfunc(n): - i = 0 - while i < n: - i += 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_JUMP_TO_TOP", uops) - - def test_jump_forward(self): - def testfunc(n): - a = 0 - while a < n: - if a < 0: - a = -a - else: - a = +a - a += 1 - return a - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - # Since there is no JUMP_FORWARD instruction, - # look for indirect evidence: the += operator - self.assertIn("_BINARY_OP_ADD_INT", uops) - - def test_for_iter_range(self): - def testfunc(n): - total = 0 - for i in range(n): - total += i - return total - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - total = testfunc(20) - self.assertEqual(total, 190) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_for_iter_list(self): - def testfunc(a): - total = 0 - for i in a: - total += i - return total - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - a = list(range(20)) - total = testfunc(a) - self.assertEqual(total, 190) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_for_iter_tuple(self): - def testfunc(a): - total = 0 - for i in a: - total += i - return total - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - a = tuple(range(20)) - total = testfunc(a) - self.assertEqual(total, 190) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_list_edge_case(self): - def testfunc(it): - for x in it: - pass - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - a = [1, 2, 3] - it = iter(a) - testfunc(it) - a.append(4) - with self.assertRaises(StopIteration): - next(it) - - def test_call_py_exact_args(self): - def testfunc(n): - def dummy(x): - return x+1 - for i in range(n): - dummy(i) - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_PUSH_FRAME", uops) - self.assertIn("_BINARY_OP_ADD_INT", uops) - - def test_branch_taken(self): - def testfunc(n): - for i in range(n): - if i < 0: - i = 0 - else: - i = 1 - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_GUARD_IS_FALSE_POP", uops) - - def test_for_iter_tier_two(self): - class MyIter: - def __init__(self, n): - self.n = n - def __iter__(self): - return self - def __next__(self): - self.n -= 1 - if self.n < 0: - raise StopIteration - return self.n - - def testfunc(n, m): - x = 0 - for i in range(m): - for j in MyIter(n): - x += 1000*i + j - return x - - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - x = testfunc(10, 10) - - self.assertEqual(x, sum(range(10)) * 10010) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} - self.assertIn("_FOR_ITER_TIER_TWO", uops) + class GetThreadId(threading.Thread): + def __init__(self): + super().__init__() + self.get_lock = threading.Lock() + self.get_lock.acquire() + self.started_lock = threading.Event() + self.py_tid = None + + def run(self): + self.started_lock.set() + self.get_lock.acquire() + self.py_tid = py_thread_id() + time.sleep(short_sleep) + self.py_tid2 = py_thread_id() + + nthread = 5 + threads = [GetThreadId() for _ in range(nthread)] + + # first make run sure that all threads are running + for thread in threads: + thread.start() + for thread in threads: + thread.started_lock.wait() + + # call _Py_ThreadId() in the main thread + py_thread_ids = [py_thread_id()] + + # now call _Py_ThreadId() in each thread + for thread in threads: + thread.get_lock.release() + + # call _Py_ThreadId() in each thread and wait until threads complete + for thread in threads: + thread.join() + py_thread_ids.append(thread.py_tid) + # _PyThread_Id() should not change for a given thread. + # For example, it should remain the same after a short sleep. + self.assertEqual(thread.py_tid2, thread.py_tid) + + # make sure that all _Py_ThreadId() are unique + for tid in py_thread_ids: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + self.assertEqual(len(set(py_thread_ids)), len(py_thread_ids), + py_thread_ids) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py new file mode 100644 index 00000000000000..5c8c0596610303 --- /dev/null +++ b/Lib/test/test_capi/test_opt.py @@ -0,0 +1,545 @@ +import contextlib +import opcode +import textwrap +import unittest + +import _testinternalcapi + + +@contextlib.contextmanager +def temporary_optimizer(opt): + old_opt = _testinternalcapi.get_optimizer() + _testinternalcapi.set_optimizer(opt) + try: + yield + finally: + _testinternalcapi.set_optimizer(old_opt) + + +@contextlib.contextmanager +def clear_executors(func): + # Clear executors in func before and after running a block + func.__code__ = func.__code__.replace() + try: + yield + finally: + func.__code__ = func.__code__.replace() + + +class TestOptimizerAPI(unittest.TestCase): + + def test_get_counter_optimizer_dealloc(self): + # See gh-108727 + def f(): + _testinternalcapi.get_counter_optimizer() + + f() + + def test_get_set_optimizer(self): + old = _testinternalcapi.get_optimizer() + opt = _testinternalcapi.get_counter_optimizer() + try: + _testinternalcapi.set_optimizer(opt) + self.assertEqual(_testinternalcapi.get_optimizer(), opt) + _testinternalcapi.set_optimizer(None) + self.assertEqual(_testinternalcapi.get_optimizer(), None) + finally: + _testinternalcapi.set_optimizer(old) + + + def test_counter_optimizer(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def loop(): + for _ in range(1000): + pass + """), ns, ns) + loop = ns['loop'] + + for repeat in range(5): + opt = _testinternalcapi.get_counter_optimizer() + with temporary_optimizer(opt): + self.assertEqual(opt.get_count(), 0) + with clear_executors(loop): + loop() + self.assertEqual(opt.get_count(), 1000) + + def test_long_loop(self): + "Check that we aren't confused by EXTENDED_ARG" + + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def nop(): + pass + + def long_loop(): + for _ in range(10): + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); + """), ns, ns) + long_loop = ns['long_loop'] + + opt = _testinternalcapi.get_counter_optimizer() + with temporary_optimizer(opt): + self.assertEqual(opt.get_count(), 0) + long_loop() + self.assertEqual(opt.get_count(), 10) + + def test_code_restore_for_ENTER_EXECUTOR(self): + def testfunc(x): + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.get_counter_optimizer() + with temporary_optimizer(opt): + testfunc(1000) + code, replace_code = testfunc.__code__, testfunc.__code__.replace() + self.assertEqual(code, replace_code) + self.assertEqual(hash(code), hash(replace_code)) + + +def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] + for i in range(0, len(co_code), 2): + if co_code[i] == JUMP_BACKWARD: + try: + return _testinternalcapi.get_executor(code, i) + except ValueError: + pass + return None + + +class TestExecutorInvalidation(unittest.TestCase): + + def setUp(self): + self.old = _testinternalcapi.get_optimizer() + self.opt = _testinternalcapi.get_counter_optimizer() + _testinternalcapi.set_optimizer(self.opt) + + def tearDown(self): + _testinternalcapi.set_optimizer(self.old) + + def test_invalidate_object(self): + # Generate a new set of functions at each call + ns = {} + func_src = "\n".join( + f""" + def f{n}(): + for _ in range(1000): + pass + """ for n in range(5) + ) + exec(textwrap.dedent(func_src), ns, ns) + funcs = [ ns[f'f{n}'] for n in range(5)] + objects = [object() for _ in range(5)] + + for f in funcs: + f() + executors = [get_first_executor(f) for f in funcs] + # Set things up so each executor depends on the objects + # with an equal or lower index. + for i, exe in enumerate(executors): + self.assertTrue(exe.is_valid()) + for obj in objects[:i+1]: + _testinternalcapi.add_executor_dependency(exe, obj) + self.assertTrue(exe.is_valid()) + # Assert that the correct executors are invalidated + # and check that nothing crashes when we invalidate + # an executor mutliple times. + for i in (4,3,2,1,0): + _testinternalcapi.invalidate_executors(objects[i]) + for exe in executors[i:]: + self.assertFalse(exe.is_valid()) + for exe in executors[:i]: + self.assertTrue(exe.is_valid()) + + def test_uop_optimizer_invalidation(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def f(): + for i in range(1000): + pass + """), ns, ns) + f = ns['f'] + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + _testinternalcapi.invalidate_executors(f.__code__) + self.assertFalse(exe.is_valid()) + +class TestUops(unittest.TestCase): + + def test_basic_loop(self): + def testfunc(x): + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(1000) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_SET_IP", uops) + self.assertIn("_LOAD_FAST", uops) + + def test_extended_arg(self): + "Check EXTENDED_ARG handling in superblock creation" + ns = {} + exec(textwrap.dedent(""" + def many_vars(): + # 260 vars, so z9 should have index 259 + a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 + b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 + c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 + d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 + e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 + f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 + g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 + h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 + i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 + j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 + k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 + l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 + m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 + n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 + o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 + p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 + q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 + s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 + t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 + u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 + v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 + w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 + x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 + y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 + z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 + while z9 > 0: + z9 = z9 - 1 + """), ns, ns) + many_vars = ns["many_vars"] + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + ex = get_first_executor(many_vars) + self.assertIsNone(ex) + many_vars() + + ex = get_first_executor(many_vars) + self.assertIsNotNone(ex) + self.assertIn(("_LOAD_FAST", 259, 0), list(ex)) + + def test_unspecialized_unpack(self): + # An example of an unspecialized opcode + def testfunc(x): + i = 0 + while i < x: + i += 1 + a, b = {1: 2, 3: 3} + assert a == 1 and b == 3 + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_UNPACK_SEQUENCE", uops) + + def test_pop_jump_if_false(self): + def testfunc(n): + i = 0 + while i < n: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_TRUE_POP", uops) + + def test_pop_jump_if_none(self): + def testfunc(a): + for x in a: + if x is None: + x = 0 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(range(20)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) + + def test_pop_jump_if_not_none(self): + def testfunc(a): + for x in a: + x = None + if x is not None: + x = 0 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(range(20)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_NONE_POP", uops) + + def test_pop_jump_if_true(self): + def testfunc(n): + i = 0 + while not i >= n: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_FALSE_POP", uops) + + def test_jump_backward(self): + def testfunc(n): + i = 0 + while i < n: + i += 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_JUMP_TO_TOP", uops) + + def test_jump_forward(self): + def testfunc(n): + a = 0 + while a < n: + if a < 0: + a = -a + else: + a = +a + a += 1 + return a + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + # Since there is no JUMP_FORWARD instruction, + # look for indirect evidence: the += operator + self.assertIn("_BINARY_OP_ADD_INT", uops) + + def test_for_iter_range(self): + def testfunc(n): + total = 0 + for i in range(n): + total += i + return total + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + total = testfunc(20) + self.assertEqual(total, 190) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_list(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + a = list(range(20)) + total = testfunc(a) + self.assertEqual(total, 190) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_tuple(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + a = tuple(range(20)) + total = testfunc(a) + self.assertEqual(total, 190) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_list_edge_case(self): + def testfunc(it): + for x in it: + pass + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + a = [1, 2, 3] + it = iter(a) + testfunc(it) + a.append(4) + with self.assertRaises(StopIteration): + next(it) + + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + + def test_branch_taken(self): + def testfunc(n): + for i in range(n): + if i < 0: + i = 0 + else: + i = 1 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(20) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_GUARD_IS_FALSE_POP", uops) + + def test_for_iter_tier_two(self): + class MyIter: + def __init__(self, n): + self.n = n + def __iter__(self): + return self + def __next__(self): + self.n -= 1 + if self.n < 0: + raise StopIteration + return self.n + + def testfunc(n, m): + x = 0 + for i in range(m): + for j in MyIter(n): + x += 1000*i + j + return x + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + x = testfunc(10, 10) + + self.assertEqual(x, sum(range(10)) * 10010) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_FOR_ITER_TIER_TWO", uops) + + def test_confidence_score(self): + def testfunc(n): + bits = 0 + for i in range(n): + if i & 0x01: + bits += 1 + if i & 0x02: + bits += 1 + if i&0x04: + bits += 1 + if i&0x08: + bits += 1 + if i&0x10: + bits += 1 + if i&0x20: + bits += 1 + return bits + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + x = testfunc(20) + + self.assertEqual(x, 40) + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + ops = [opname for opname, _, _ in ex] + count = ops.count("_GUARD_IS_TRUE_POP") + # Because Each 'if' halves the score, the second branch is + # too much already. + self.assertEqual(count, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index da957fcebaa296..7323bdd801f4be 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -8,7 +8,6 @@ from test.support.os_helper import TESTFN, unlink from textwrap import dedent from unittest import TestCase -import contextlib import inspect import os.path import re @@ -17,12 +16,13 @@ test_tools.skip_if_missing('clinic') with test_tools.imports_under_tool('clinic'): + import libclinic import clinic from clinic import DSLParser def _make_clinic(*, filename='clinic_tests'): - clang = clinic.CLanguage(None) + clang = clinic.CLanguage(filename) c = clinic.Clinic(clang, filename=filename, limited_capi=False) c.block_parser = clinic.BlockParser('', clang) return c @@ -264,70 +264,6 @@ def converter_init(self): ) self.expect_failure(raw, err) - @staticmethod - @contextlib.contextmanager - def _clinic_version(new_version): - """Helper for test_version_*() tests""" - _saved = clinic.version - clinic.version = new_version - try: - yield - finally: - clinic.version = _saved - - def test_version_directive(self): - dataset = ( - # (clinic version, required version) - ('3', '2'), # required version < clinic version - ('3.1', '3.0'), # required version < clinic version - ('1.2b0', '1.2a7'), # required version < clinic version - ('5', '5'), # required version == clinic version - ('6.1', '6.1'), # required version == clinic version - ('1.2b3', '1.2b3'), # required version == clinic version - ) - for clinic_version, required_version in dataset: - with self.subTest(clinic_version=clinic_version, - required_version=required_version): - with self._clinic_version(clinic_version): - block = dedent(f""" - /*[clinic input] - version {required_version} - [clinic start generated code]*/ - """) - self.clinic.parse(block) - - def test_version_directive_insufficient_version(self): - with self._clinic_version('4'): - err = ( - "Insufficient Clinic version!\n" - " Version: 4\n" - " Required: 5" - ) - block = """ - /*[clinic input] - version 5 - [clinic start generated code]*/ - """ - self.expect_failure(block, err) - - def test_version_directive_illegal_char(self): - err = "Illegal character 'v' in version string 'v5'" - block = """ - /*[clinic input] - version v5 - [clinic start generated code]*/ - """ - self.expect_failure(block, err) - - def test_version_directive_unsupported_string(self): - err = "Unsupported version string: '.-'" - block = """ - /*[clinic input] - version .- - [clinic start generated code]*/ - """ - self.expect_failure(block, err) - def test_clone_mismatch(self): err = "'kind' of function and cloned function don't match!" block = """ @@ -638,7 +574,7 @@ class C "void *" "" C.__init__ = C.meth [clinic start generated code]*/ """ - err = "'__init__' must be a normal method, not a class or static method" + err = "'__init__' must be a normal method; got 'FunctionKind.CLASS_METHOD'!" self.expect_failure(block, err, lineno=8) def test_validate_cloned_new(self): @@ -2180,14 +2116,100 @@ class Foo "" "" self.expect_failure(block, err, lineno=2) def test_init_must_be_a_normal_method(self): - err = "'__init__' must be a normal method, not a class or static method!" + err_template = "'__init__' must be a normal method; got 'FunctionKind.{}'!" + annotations = { + "@classmethod": "CLASS_METHOD", + "@staticmethod": "STATIC_METHOD", + "@getter": "GETTER", + } + for annotation, invalid_kind in annotations.items(): + with self.subTest(annotation=annotation, invalid_kind=invalid_kind): + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.__init__ + """ + expected_error = err_template.format(invalid_kind) + self.expect_failure(block, expected_error, lineno=3) + + def test_invalid_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property -> int + """ + expected_error = f"{annotation} method cannot define a return type" + self.expect_failure(block, expected_error, lineno=3) + + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property + obj: int + / + """ + expected_error = f"{annotation} method cannot define parameters" + self.expect_failure(block, expected_error) + + def test_setter_docstring(self): block = """ module foo class Foo "" "" - @classmethod - Foo.__init__ + @setter + Foo.property + + foo + + bar + [clinic start generated code]*/ """ - self.expect_failure(block, err, lineno=3) + expected_error = "docstrings are only supported for @getter, not @setter" + self.expect_failure(block, expected_error) + + def test_duplicate_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + {annotation} + Foo.property -> int + """ + expected_error = f"Cannot apply {annotation} twice to the same function!" + self.expect_failure(block, expected_error, lineno=3) + + def test_getter_and_setter_disallowed_on_same_function(self): + dup_annotations = [("@getter", "@setter"), ("@setter", "@getter")] + for dup in dup_annotations: + with self.subTest(dup=dup): + block = f""" + module foo + class Foo "" "" + {dup[0]} + {dup[1]} + Foo.property -> int + """ + expected_error = "Cannot apply both @getter and @setter to the same function!" + self.expect_failure(block, expected_error, lineno=3) + + def test_getset_no_class(self): + for annotation in "@getter", "@setter": + with self.subTest(annotation=annotation): + block = f""" + module m + {annotation} + m.func + """ + expected_error = "@getter and @setter must be methods" + self.expect_failure(block, expected_error, lineno=2) def test_duplicate_coexist(self): err = "Called @coexist twice" @@ -3708,7 +3730,7 @@ def test_strip_leading_and_trailing_blank_lines(self): ) for lines, expected in dataset: with self.subTest(lines=lines, expected=expected): - out = clinic.strip_leading_and_trailing_blank_lines(lines) + out = libclinic.normalize_snippet(lines) self.assertEqual(out, expected) def test_normalize_snippet(self): @@ -3737,63 +3759,33 @@ def test_normalize_snippet(self): expected_outputs = {0: zero_indent, 4: four_indent, 8: eight_indent} for indent, expected in expected_outputs.items(): with self.subTest(indent=indent): - actual = clinic.normalize_snippet(snippet, indent=indent) + actual = libclinic.normalize_snippet(snippet, indent=indent) self.assertEqual(actual, expected) - def test_accumulator(self): - acc = clinic.text_accumulator() - self.assertEqual(acc.output(), "") - acc.append("a") - self.assertEqual(acc.output(), "a") - self.assertEqual(acc.output(), "") - acc.append("b") - self.assertEqual(acc.output(), "b") - self.assertEqual(acc.output(), "") - acc.append("c") - acc.append("d") - self.assertEqual(acc.output(), "cd") - self.assertEqual(acc.output(), "") - - def test_quoted_for_c_string(self): + def test_escaped_docstring(self): dataset = ( # input, expected - (r"abc", r"abc"), - (r"\abc", r"\\abc"), - (r"\a\bc", r"\\a\\bc"), - (r"\a\\bc", r"\\a\\\\bc"), - (r'"abc"', r'\"abc\"'), - (r"'a'", r"\'a\'"), + (r"abc", r'"abc"'), + (r"\abc", r'"\\abc"'), + (r"\a\bc", r'"\\a\\bc"'), + (r"\a\\bc", r'"\\a\\\\bc"'), + (r'"abc"', r'"\"abc\""'), + (r"'a'", r'"\'a\'"'), ) for line, expected in dataset: with self.subTest(line=line, expected=expected): - out = clinic.quoted_for_c_string(line) + out = libclinic.docstring_for_c_string(line) self.assertEqual(out, expected) - def test_rstrip_lines(self): - lines = ( - "a \n" - "b\n" - " c\n" - " d \n" - ) - expected = ( - "a\n" - "b\n" - " c\n" - " d\n" - ) - out = clinic.rstrip_lines(lines) - self.assertEqual(out, expected) - def test_format_escape(self): line = "{}, {a}" expected = "{{}}, {{a}}" - out = clinic.format_escape(line) + out = libclinic.format_escape(line) self.assertEqual(out, expected) def test_indent_all_lines(self): # Blank lines are expected to be unchanged. - self.assertEqual(clinic.indent_all_lines("", prefix="bar"), "") + self.assertEqual(libclinic.indent_all_lines("", prefix="bar"), "") lines = ( "one\n" @@ -3803,7 +3795,7 @@ def test_indent_all_lines(self): "barone\n" "bartwo" ) - out = clinic.indent_all_lines(lines, prefix="bar") + out = libclinic.indent_all_lines(lines, prefix="bar") self.assertEqual(out, expected) # If last line is empty, expect it to be unchanged. @@ -3819,12 +3811,12 @@ def test_indent_all_lines(self): "bartwo\n" "" ) - out = clinic.indent_all_lines(lines, prefix="bar") + out = libclinic.indent_all_lines(lines, prefix="bar") self.assertEqual(out, expected) def test_suffix_all_lines(self): # Blank lines are expected to be unchanged. - self.assertEqual(clinic.suffix_all_lines("", suffix="foo"), "") + self.assertEqual(libclinic.suffix_all_lines("", suffix="foo"), "") lines = ( "one\n" @@ -3834,7 +3826,7 @@ def test_suffix_all_lines(self): "onefoo\n" "twofoo" ) - out = clinic.suffix_all_lines(lines, suffix="foo") + out = libclinic.suffix_all_lines(lines, suffix="foo") self.assertEqual(out, expected) # If last line is empty, expect it to be unchanged. @@ -3850,7 +3842,7 @@ def test_suffix_all_lines(self): "twofoo\n" "" ) - out = clinic.suffix_all_lines(lines, suffix="foo") + out = libclinic.suffix_all_lines(lines, suffix="foo") self.assertEqual(out, expected) @@ -3928,7 +3920,7 @@ def test_Function_and_Parameter_reprs(self): self.assertEqual(repr(parameter), "") def test_Monitor_repr(self): - monitor = clinic.cpp.Monitor() + monitor = libclinic.cpp.Monitor("test.c") self.assertRegex(repr(monitor), r"") monitor.line_number = 42 diff --git a/Lib/test/test_cmd.py b/Lib/test/test_cmd.py index 951336fa08542d..46ec82b704963d 100644 --- a/Lib/test/test_cmd.py +++ b/Lib/test/test_cmd.py @@ -9,7 +9,10 @@ import doctest import unittest import io +import textwrap from test import support +from test.support.import_helper import import_module +from test.support.pty_helper import run_pty class samplecmdclass(cmd.Cmd): """ @@ -259,6 +262,33 @@ class CmdPrintExceptionClass(cmd.Cmd): def default(self, line): print(sys.exc_info()[:2]) + +@support.requires_subprocess() +class CmdTestReadline(unittest.TestCase): + def setUpClass(): + # Ensure that the readline module is loaded + # If this fails, the test is skipped because SkipTest will be raised + readline = import_module('readline') + + def test_basic_completion(self): + script = textwrap.dedent(""" + import cmd + class simplecmd(cmd.Cmd): + def do_tab_completion_test(self, args): + print('tab completion success') + return True + + simplecmd().cmdloop() + """) + + # 't' and complete 'ab_completion_test' to 'tab_completion_test' + input = b"t\t\n" + + output = run_pty(script, input) + + self.assertIn(b'ab_completion_test', output) + self.assertIn(b'tab completion success', output) + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) return tests diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 7a27952c345b9c..1fe3b2fe53c0b6 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -738,6 +738,8 @@ def test_xdev(self): out = self.run_xdev("-c", code, check_exitcode=False) if support.with_pymalloc(): alloc_name = "pymalloc_debug" + elif support.Py_GIL_DISABLED: + alloc_name = "mimalloc_debug" else: alloc_name = "malloc_debug" self.assertEqual(out, alloc_name) @@ -814,9 +816,13 @@ def check_pythonmalloc(self, env_var, name): @support.cpython_only def test_pythonmalloc(self): # Test the PYTHONMALLOC environment variable + malloc = not support.Py_GIL_DISABLED pymalloc = support.with_pymalloc() mimalloc = support.with_mimalloc() - if pymalloc: + if support.Py_GIL_DISABLED: + default_name = 'mimalloc_debug' if support.Py_DEBUG else 'mimalloc' + default_name_debug = 'mimalloc_debug' + elif pymalloc: default_name = 'pymalloc_debug' if support.Py_DEBUG else 'pymalloc' default_name_debug = 'pymalloc_debug' else: @@ -826,9 +832,12 @@ def test_pythonmalloc(self): tests = [ (None, default_name), ('debug', default_name_debug), - ('malloc', 'malloc'), - ('malloc_debug', 'malloc_debug'), ] + if malloc: + tests.extend([ + ('malloc', 'malloc'), + ('malloc_debug', 'malloc_debug'), + ]) if pymalloc: tests.extend(( ('pymalloc', 'pymalloc'), diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index a961ddbe17a3d3..d8fb826edeb681 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -144,6 +144,8 @@ gc_collect) from test.support.script_helper import assert_python_ok from test.support import threading_helper +from test.support.bytecode_helper import (BytecodeTestCase, + instructions_with_positions) from opcode import opmap, opname COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -384,10 +386,8 @@ def test_co_positions_artificial_instructions(self): code = traceback.tb_frame.f_code artificial_instructions = [] - for instr, positions in zip( - dis.get_instructions(code, show_caches=True), - code.co_positions(), - strict=True + for instr, positions in instructions_with_positions( + dis.get_instructions(code), code.co_positions() ): # If any of the positions is None, then all have to # be None as well for the case above. There are still diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index bb8b352518ef3e..7e6f811e17cfa2 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -488,12 +488,8 @@ def test_instance(self): self.assertEqual(p._replace(x=1), (1, 22)) # test _replace method self.assertEqual(p._asdict(), dict(x=11, y=22)) # test _asdict method - try: + with self.assertRaises(TypeError): p._replace(x=1, error=2) - except ValueError: - pass - else: - self._fail('Did not detect an incorrect fieldname') # verify that field string can have commas Point = namedtuple('Point', 'x, y') diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index df6e5e4b55f728..7850977428985f 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -12,6 +12,7 @@ from test import support from test.support import (script_helper, requires_debug_ranges, requires_specialization, Py_C_RECURSION_LIMIT) +from test.support.bytecode_helper import instructions_with_positions from test.support.os_helper import FakePath class TestSpecifics(unittest.TestCase): @@ -443,6 +444,23 @@ def f(): self.assertIn("_A__mangled_mod", A.f.__code__.co_varnames) self.assertIn("__package__", A.f.__code__.co_varnames) + def test_condition_expression_with_dead_blocks_compiles(self): + # See gh-113054 + compile('if (5 if 5 else T): 0', '', 'exec') + + def test_condition_expression_with_redundant_comparisons_compiles(self): + # See gh-113054 + compile('if 9<9<9and 9or 9:9', '', 'exec') + + def test_dead_code_with_except_handler_compiles(self): + compile(textwrap.dedent(""" + if None: + with CM: + x = 1 + else: + x = 2 + """), '', 'exec') + def test_compile_invalid_namedexpr(self): # gh-109351 m = ast.Module( @@ -1346,8 +1364,8 @@ def generic_visit(self, node): def assertOpcodeSourcePositionIs(self, code, opcode, line, end_line, column, end_column, occurrence=1): - for instr, position in zip( - dis.Bytecode(code, show_caches=True), code.co_positions(), strict=True + for instr, position in instructions_with_positions( + dis.Bytecode(code), code.co_positions() ): if instr.opname == opcode: occurrence -= 1 diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 9180cca62b28b8..b057121f285dc7 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -109,6 +109,8 @@ def test_truediv(self): complex(random(), random())) self.assertAlmostEqual(complex.__truediv__(2+0j, 1+1j), 1-1j) + self.assertRaises(TypeError, operator.truediv, 1j, None) + self.assertRaises(TypeError, operator.truediv, None, 1j) for denom_real, denom_imag in [(0, NAN), (NAN, 0), (NAN, NAN)]: z = complex(0, 0) / complex(denom_real, denom_imag) @@ -140,6 +142,7 @@ def test_floordiv_zero_division(self): def test_richcompare(self): self.assertIs(complex.__eq__(1+1j, 1<<10000), False) self.assertIs(complex.__lt__(1+1j, None), NotImplemented) + self.assertIs(complex.__eq__(1+1j, None), NotImplemented) self.assertIs(complex.__eq__(1+1j, 1+1j), True) self.assertIs(complex.__eq__(1+1j, 2+2j), False) self.assertIs(complex.__ne__(1+1j, 1+1j), False) @@ -162,6 +165,7 @@ def test_richcompare(self): self.assertIs(operator.eq(1+1j, 2+2j), False) self.assertIs(operator.ne(1+1j, 1+1j), False) self.assertIs(operator.ne(1+1j, 2+2j), True) + self.assertIs(operator.eq(1+1j, 2.0), False) def test_richcompare_boundaries(self): def check(n, deltas, is_equal, imag = 0.0): @@ -180,6 +184,27 @@ def check(n, deltas, is_equal, imag = 0.0): check(2 ** pow, range(1, 101), lambda delta: False, float(i)) check(2 ** 53, range(-100, 0), lambda delta: True) + def test_add(self): + self.assertEqual(1j + int(+1), complex(+1, 1)) + self.assertEqual(1j + int(-1), complex(-1, 1)) + self.assertRaises(OverflowError, operator.add, 1j, 10**1000) + self.assertRaises(TypeError, operator.add, 1j, None) + self.assertRaises(TypeError, operator.add, None, 1j) + + def test_sub(self): + self.assertEqual(1j - int(+1), complex(-1, 1)) + self.assertEqual(1j - int(-1), complex(1, 1)) + self.assertRaises(OverflowError, operator.sub, 1j, 10**1000) + self.assertRaises(TypeError, operator.sub, 1j, None) + self.assertRaises(TypeError, operator.sub, None, 1j) + + def test_mul(self): + self.assertEqual(1j * int(20), complex(0, 20)) + self.assertEqual(1j * int(-1), complex(0, -1)) + self.assertRaises(OverflowError, operator.mul, 1j, 10**1000) + self.assertRaises(TypeError, operator.mul, 1j, None) + self.assertRaises(TypeError, operator.mul, None, 1j) + def test_mod(self): # % is no longer supported on complex numbers with self.assertRaises(TypeError): @@ -212,11 +237,18 @@ def test_divmod_zero_division(self): def test_pow(self): self.assertAlmostEqual(pow(1+1j, 0+0j), 1.0) self.assertAlmostEqual(pow(0+0j, 2+0j), 0.0) + self.assertEqual(pow(0+0j, 2000+0j), 0.0) + self.assertEqual(pow(0, 0+0j), 1.0) + self.assertEqual(pow(-1, 0+0j), 1.0) self.assertRaises(ZeroDivisionError, pow, 0+0j, 1j) + self.assertRaises(ZeroDivisionError, pow, 0+0j, -1000) self.assertAlmostEqual(pow(1j, -1), 1/1j) self.assertAlmostEqual(pow(1j, 200), 1) self.assertRaises(ValueError, pow, 1+1j, 1+1j, 1+1j) self.assertRaises(OverflowError, pow, 1e200+1j, 1e200+1j) + self.assertRaises(TypeError, pow, 1j, None) + self.assertRaises(TypeError, pow, None, 1j) + self.assertAlmostEqual(pow(1j, 0.5), 0.7071067811865476+0.7071067811865475j) a = 3.33+4.43j self.assertEqual(a ** 0j, 1) @@ -301,6 +333,7 @@ def test_boolcontext(self): for i in range(100): self.assertTrue(complex(random() + 1e-6, random() + 1e-6)) self.assertTrue(not complex(0.0, 0.0)) + self.assertTrue(1j) def test_conjugate(self): self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) @@ -314,6 +347,8 @@ def __complex__(self): return self.value self.assertRaises(TypeError, complex, {}) self.assertRaises(TypeError, complex, NS(1.5)) self.assertRaises(TypeError, complex, NS(1)) + self.assertRaises(TypeError, complex, object()) + self.assertRaises(TypeError, complex, NS(4.25+0.5j), object()) self.assertAlmostEqual(complex("1+10j"), 1+10j) self.assertAlmostEqual(complex(10), 10+0j) @@ -359,6 +394,8 @@ def __complex__(self): return self.value self.assertAlmostEqual(complex('1e-500'), 0.0 + 0.0j) self.assertAlmostEqual(complex('-1e-500j'), 0.0 - 0.0j) self.assertAlmostEqual(complex('-1e-500+1e-500j'), -0.0 + 0.0j) + self.assertEqual(complex('1-1j'), 1.0 - 1j) + self.assertEqual(complex('1J'), 1j) class complex2(complex): pass self.assertAlmostEqual(complex(complex2(1+1j)), 1+1j) @@ -553,6 +590,8 @@ def test_hash(self): x /= 3.0 # now check against floating point self.assertEqual(hash(x), hash(complex(x, 0.))) + self.assertNotEqual(hash(2000005 - 1j), -1) + def test_abs(self): nums = [complex(x/3., y/7.) for x in range(-9,9) for y in range(-9,9)] for num in nums: @@ -602,6 +641,14 @@ def test(v, expected, test_fn=self.assertEqual): test(complex(-0., 0.), "(-0+0j)") test(complex(-0., -0.), "(-0-0j)") + def test_pos(self): + class ComplexSubclass(complex): + pass + + self.assertEqual(+(1+6j), 1+6j) + self.assertEqual(+ComplexSubclass(1, 6), 1+6j) + self.assertIs(type(+ComplexSubclass(1, 6)), complex) + def test_neg(self): self.assertEqual(-(1+6j), -1-6j) diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 60735ba89a80ee..89102373759ca0 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -952,7 +952,7 @@ class PointFromClass(NamedTuple): self.assertEqual(copy.replace(p, x=1), (1, 22)) self.assertEqual(copy.replace(p, y=2), (11, 2)) self.assertEqual(copy.replace(p, x=1, y=2), (1, 2)) - with self.assertRaisesRegex(ValueError, 'unexpected field name'): + with self.assertRaisesRegex(TypeError, 'unexpected field name'): copy.replace(p, x=1, error=2) def test_dataclass(self): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 47145782c0f04f..d848bfbd46c83b 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -953,11 +953,12 @@ async def b(): def test_corotype_1(self): ct = types.CoroutineType - self.assertIn('into coroutine', ct.send.__doc__) - self.assertIn('inside coroutine', ct.close.__doc__) - self.assertIn('in coroutine', ct.throw.__doc__) - self.assertIn('of the coroutine', ct.__dict__['__name__'].__doc__) - self.assertIn('of the coroutine', ct.__dict__['__qualname__'].__doc__) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn('into coroutine', ct.send.__doc__) + self.assertIn('inside coroutine', ct.close.__doc__) + self.assertIn('in coroutine', ct.throw.__doc__) + self.assertIn('of the coroutine', ct.__dict__['__name__'].__doc__) + self.assertIn('of the coroutine', ct.__dict__['__qualname__'].__doc__) self.assertEqual(ct.__name__, 'coroutine') async def f(): pass @@ -2216,6 +2217,14 @@ async def f(): gen.cr_frame.clear() gen.close() + def test_cr_frame_after_close(self): + async def f(): + pass + gen = f() + self.assertIsNotNone(gen.cr_frame) + gen.close() + self.assertIsNone(gen.cr_frame) + def test_stack_in_coroutine_throw(self): # Regression test for https://github.com/python/cpython/issues/93592 async def a(): diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 299a16ada2e32e..c6039bd17b0662 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -2,7 +2,6 @@ # compatible with C++ and does not emit C++ compiler warnings. import os.path import shutil -import sys import unittest import subprocess import sysconfig @@ -15,7 +14,7 @@ # gh-110119: pip does not currently support 't' in the ABI flag use by # --disable-gil builds. Once it does, we can remove this skip. -@unittest.skipIf(sysconfig.get_config_var('Py_GIL_DISABLED') == 1, +@unittest.skipIf(support.Py_GIL_DISABLED, 'test does not work with --disable-gil') @support.requires_subprocess() class TestCPPExt(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index f05ee5e491a41e..3eafc77ca70aea 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -1,13 +1,16 @@ import _ctypes_test +from platform import architecture as _architecture import struct import sys import unittest -from ctypes import (CDLL, Structure, Union, POINTER, sizeof, byref, alignment, +from ctypes import (CDLL, Array, Structure, Union, POINTER, sizeof, byref, alignment, c_void_p, c_char, c_wchar, c_byte, c_ubyte, c_uint8, c_uint16, c_uint32, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from ctypes.util import find_library from struct import calcsize +from collections import namedtuple from test import support @@ -470,37 +473,116 @@ class X(Structure): self.assertEqual(s.first, got.first) self.assertEqual(s.second, got.second) + def _test_issue18060(self, Vector): + # The call to atan2() should succeed if the + # class fields were correctly cloned in the + # subclasses. Otherwise, it will segfault. + if sys.platform == 'win32': + libm = CDLL(find_library('msvcrt.dll')) + else: + libm = CDLL(find_library('m')) + + libm.atan2.argtypes = [Vector] + libm.atan2.restype = c_double + + arg = Vector(y=0.0, x=-1.0) + self.assertAlmostEqual(libm.atan2(arg), 3.141592653589793) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_a(self): + # This test case calls + # PyCStructUnionType_update_stgdict() for each + # _fields_ assignment, and PyCStgDict_clone() + # for the Mid and Vector class definitions. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + class Mid(Base): + pass + Mid._fields_ = [] + class Vector(Mid): pass + self._test_issue18060(Vector) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_b(self): + # This test case calls + # PyCStructUnionType_update_stgdict() for each + # _fields_ assignment. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + class Mid(Base): + _fields_ = [] + class Vector(Mid): + _fields_ = [] + self._test_issue18060(Vector) + + @unittest.skipIf(_architecture() == ('64bit', 'WindowsPE'), "can't test Windows x64 build") + @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") + def test_issue18060_c(self): + # This test case calls + # PyCStructUnionType_update_stgdict() for each + # _fields_ assignment. + class Base(Structure): + _fields_ = [('y', c_double)] + class Mid(Base): + _fields_ = [] + class Vector(Mid): + _fields_ = [('x', c_double)] + self._test_issue18060(Vector) + def test_array_in_struct(self): # See bpo-22273 + # Load the shared library + dll = CDLL(_ctypes_test.__file__) + # These should mirror the structures in Modules/_ctypes/_ctypes_test.c class Test2(Structure): _fields_ = [ ('data', c_ubyte * 16), ] - class Test3(Structure): + class Test3AParent(Structure): + _fields_ = [ + ('data', c_float * 2), + ] + + class Test3A(Test3AParent): + _fields_ = [ + ('more_data', c_float * 2), + ] + + class Test3B(Structure): _fields_ = [ ('data', c_double * 2), ] - class Test3A(Structure): + class Test3C(Structure): _fields_ = [ - ('data', c_float * 2), + ("data", c_double * 4) ] - class Test3B(Test3A): + class Test3D(Structure): _fields_ = [ - ('more_data', c_float * 2), + ("data", c_double * 8) ] + class Test3E(Structure): + _fields_ = [ + ("data", c_double * 9) + ] + + + # Tests for struct Test2 s = Test2() expected = 0 for i in range(16): s.data[i] = i expected += i - dll = CDLL(_ctypes_test.__file__) - func = dll._testfunc_array_in_struct1 + func = dll._testfunc_array_in_struct2 func.restype = c_int func.argtypes = (Test2,) result = func(s) @@ -509,29 +591,16 @@ class Test3B(Test3A): for i in range(16): self.assertEqual(s.data[i], i) - s = Test3() - s.data[0] = 3.14159 - s.data[1] = 2.71828 - expected = 3.14159 + 2.71828 - func = dll._testfunc_array_in_struct2 - func.restype = c_double - func.argtypes = (Test3,) - result = func(s) - self.assertEqual(result, expected) - # check the passed-in struct hasn't changed - self.assertEqual(s.data[0], 3.14159) - self.assertEqual(s.data[1], 2.71828) - - s = Test3B() + # Tests for struct Test3A + s = Test3A() s.data[0] = 3.14159 s.data[1] = 2.71828 s.more_data[0] = -3.0 s.more_data[1] = -2.0 - - expected = 3.14159 + 2.71828 - 5.0 - func = dll._testfunc_array_in_struct2a + expected = 3.14159 + 2.71828 - 3.0 - 2.0 + func = dll._testfunc_array_in_struct3A func.restype = c_double - func.argtypes = (Test3B,) + func.argtypes = (Test3A,) result = func(s) self.assertAlmostEqual(result, expected, places=6) # check the passed-in struct hasn't changed @@ -540,6 +609,61 @@ class Test3B(Test3A): self.assertAlmostEqual(s.more_data[0], -3.0, places=6) self.assertAlmostEqual(s.more_data[1], -2.0, places=6) + # Test3B, Test3C, Test3D, Test3E have the same logic with different + # sizes hence putting them in a loop. + StructCtype = namedtuple( + "StructCtype", + ["cls", "cfunc1", "cfunc2", "items"] + ) + structs_to_test = [ + StructCtype( + Test3B, + dll._testfunc_array_in_struct3B, + dll._testfunc_array_in_struct3B_set_defaults, + 2), + StructCtype( + Test3C, + dll._testfunc_array_in_struct3C, + dll._testfunc_array_in_struct3C_set_defaults, + 4), + StructCtype( + Test3D, + dll._testfunc_array_in_struct3D, + dll._testfunc_array_in_struct3D_set_defaults, + 8), + StructCtype( + Test3E, + dll._testfunc_array_in_struct3E, + dll._testfunc_array_in_struct3E_set_defaults, + 9), + ] + + for sut in structs_to_test: + s = sut.cls() + + # Test for cfunc1 + expected = 0 + for i in range(sut.items): + float_i = float(i) + s.data[i] = float_i + expected += float_i + func = sut.cfunc1 + func.restype = c_double + func.argtypes = (sut.cls,) + result = func(s) + self.assertEqual(result, expected) + # check the passed-in struct hasn't changed + for i in range(sut.items): + self.assertEqual(s.data[i], float(i)) + + # Test for cfunc2 + func = sut.cfunc2 + func.restype = sut.cls + result = func() + # check if the default values have been set correctly + for i in range(sut.items): + self.assertEqual(result.data[i], float(i+1)) + def test_38368(self): class U(Union): _fields_ = [ diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 31bc108e7712ea..83d10dd8579074 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock from test.support import (requires, verbose, SaveSignals, cpython_only, - check_disallow_instantiation) + check_disallow_instantiation, MISSING_C_DOCSTRINGS) from test.support.import_helper import import_module # Optionally test curses module. This currently requires that the @@ -1142,6 +1142,8 @@ def test_encoding(self): with self.assertRaises(TypeError): del stdscr.encoding + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_issue21088(self): stdscr = self.stdscr # diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 4a3db80ca43c27..fd0af9b30a0a71 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5004,6 +5004,21 @@ class Child(Parent): gc.collect() self.assertEqual(Parent.__subclasses__(), []) + def test_instance_method_get_behavior(self): + # test case for gh-113157 + + class A: + def meth(self): + return self + + class B: + pass + + a = A() + b = B() + b.meth = a.meth.__get__(b, B) + self.assertEqual(b.meth(), a) + def test_attr_raise_through_property(self): # test case for gh-103272 class A: diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 805cd4e4c30965..e4c7e9ba4649f7 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -2,6 +2,7 @@ import contextlib import dis +import functools import io import re import sys @@ -13,6 +14,7 @@ import opcode +CACHE = dis.opmap["CACHE"] def get_tb(): def _error(): @@ -1208,8 +1210,13 @@ def test_loop_quicken(self): got = self.get_disassembly(loop_test, adaptive=True) expected = dis_loop_test_quickened_code if _testinternalcapi.get_optimizer(): - # We *may* see ENTER_EXECUTOR in the disassembly - got = got.replace("ENTER_EXECUTOR", "JUMP_BACKWARD ") + # We *may* see ENTER_EXECUTOR in the disassembly. This is a + # temporary hack to keep the test working until dis is able to + # handle the instruction correctly (GH-112383): + got = got.replace( + "ENTER_EXECUTOR 16", + "JUMP_BACKWARD 16 (to L1)", + ) self.do_disassembly_compare(got, expected) @cpython_only @@ -1227,9 +1234,9 @@ def f(): else: # "copy" the code to un-quicken it: f.__code__ = f.__code__.replace() - for instruction in dis.get_instructions( + for instruction in _unroll_caches_as_Instructions(dis.get_instructions( f, show_caches=True, adaptive=adaptive - ): + ), show_caches=True): if instruction.opname == "CACHE": yield instruction.argrepr @@ -1262,7 +1269,8 @@ def f(): # However, this might change in the future. So we explicitly try to find # a CACHE entry in the instructions. If we can't do that, fail the test - for inst in dis.get_instructions(f, show_caches=True): + for inst in _unroll_caches_as_Instructions( + dis.get_instructions(f, show_caches=True), show_caches=True): if inst.opname == "CACHE": op_offset = inst.offset - 2 cache_offset = inst.offset @@ -1775,8 +1783,8 @@ def simple(): pass class InstructionTestCase(BytecodeTestCase): def assertInstructionsEqual(self, instrs_1, instrs_2, /): - instrs_1 = [instr_1._replace(positions=None) for instr_1 in instrs_1] - instrs_2 = [instr_2._replace(positions=None) for instr_2 in instrs_2] + instrs_1 = [instr_1._replace(positions=None, cache_info=None) for instr_1 in instrs_1] + instrs_2 = [instr_2._replace(positions=None, cache_info=None) for instr_2 in instrs_2] self.assertEqual(instrs_1, instrs_2) class InstructionTests(InstructionTestCase): @@ -1785,6 +1793,12 @@ def __init__(self, *args): super().__init__(*args) self.maxDiff = None + def test_instruction_str(self): + # smoke test for __str__ + instrs = dis.get_instructions(simple) + for instr in instrs: + str(instr) + def test_default_first_line(self): actual = dis.get_instructions(simple) self.assertInstructionsEqual(list(actual), expected_opinfo_simple) @@ -1884,9 +1898,9 @@ def roots(a, b, c): instruction.positions.col_offset, instruction.positions.end_col_offset, ) - for instruction in dis.get_instructions( + for instruction in _unroll_caches_as_Instructions(dis.get_instructions( code, adaptive=adaptive, show_caches=show_caches - ) + ), show_caches=show_caches) ] self.assertEqual(co_positions, dis_positions) @@ -1896,6 +1910,18 @@ def test_oparg_alias(self): positions=None) self.assertEqual(instruction.arg, instruction.oparg) + def test_show_caches_with_label(self): + def f(x, y, z): + if x: + res = y + else: + res = z + return res + + output = io.StringIO() + dis.dis(f.__code__, file=output, show_caches=True) + self.assertIn("L1:", output.getvalue()) + def test_baseopname_and_baseopcode(self): # Standard instructions for name, code in dis.opmap.items(): @@ -1943,15 +1969,16 @@ def test_jump_target(self): self.assertEqual(10 + 2 + 1*2 + 100*2, instruction.jump_target) def test_argval_argrepr(self): - def f(*args): - return dis.Instruction._get_argval_argrepr( - *args, labels_map={24: 1}) + def f(opcode, oparg, offset, *init_args): + arg_resolver = dis.ArgResolver(*init_args) + return arg_resolver.get_argval_argrepr(opcode, oparg, offset) offset = 42 co_consts = (0, 1, 2, 3) names = {1: 'a', 2: 'b'} varname_from_oparg = lambda i : names[i] - args = (offset, co_consts, names, varname_from_oparg) + labels_map = {24: 1} + args = (offset, co_consts, names, varname_from_oparg, labels_map) self.assertEqual(f(opcode.opmap["POP_TOP"], None, *args), (None, '')) self.assertEqual(f(opcode.opmap["LOAD_CONST"], 1, *args), (1, '1')) self.assertEqual(f(opcode.opmap["LOAD_GLOBAL"], 2, *args), ('a', 'a')) @@ -1961,19 +1988,27 @@ def f(*args): self.assertEqual(f(opcode.opmap["BINARY_OP"], 3, *args), (3, '<<')) self.assertEqual(f(opcode.opmap["CALL_INTRINSIC_1"], 2, *args), (2, 'INTRINSIC_IMPORT_STAR')) + def get_instructions(self, code): + return dis._get_instructions_bytes(code) + def test_start_offset(self): # When no extended args are present, # start_offset should be equal to offset + instructions = list(dis.Bytecode(_f)) for instruction in instructions: self.assertEqual(instruction.offset, instruction.start_offset) + def last_item(iterable): + return functools.reduce(lambda a, b : b, iterable) + code = bytes([ opcode.opmap["LOAD_FAST"], 0x00, opcode.opmap["EXTENDED_ARG"], 0x01, opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, ]) - jump = list(dis._get_instructions_bytes(code))[-1] + labels_map = dis._make_labels_map(code) + jump = last_item(self.get_instructions(code)) self.assertEqual(4, jump.offset) self.assertEqual(2, jump.start_offset) @@ -1985,7 +2020,7 @@ def test_start_offset(self): opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, opcode.opmap["CACHE"], 0x00, ]) - jump = list(dis._get_instructions_bytes(code))[-1] + jump = last_item(self.get_instructions(code)) self.assertEqual(8, jump.offset) self.assertEqual(2, jump.start_offset) @@ -2000,7 +2035,7 @@ def test_start_offset(self): opcode.opmap["POP_JUMP_IF_TRUE"], 0xFF, opcode.opmap["CACHE"], 0x00, ]) - instructions = list(dis._get_instructions_bytes(code)) + instructions = list(self.get_instructions(code)) # 1st jump self.assertEqual(4, instructions[2].offset) self.assertEqual(2, instructions[2].start_offset) @@ -2021,7 +2056,7 @@ def test_cache_offset_and_end_offset(self): opcode.opmap["CACHE"], 0x00, opcode.opmap["CACHE"], 0x00 ]) - instructions = list(dis._get_instructions_bytes(code)) + instructions = list(self.get_instructions(code)) self.assertEqual(2, instructions[0].cache_offset) self.assertEqual(10, instructions[0].end_offset) self.assertEqual(12, instructions[1].cache_offset) @@ -2214,6 +2249,31 @@ def get_disassembly(self, tb): dis.distb(tb, file=output) return output.getvalue() +def _unroll_caches_as_Instructions(instrs, show_caches=False): + # Cache entries are no longer reported by dis as fake instructions, + # but some tests assume that do. We should rewrite the tests to assume + # the new API, but it will be clearer to keep the tests working as + # before and do that in a separate PR. + + for instr in instrs: + yield instr + if not show_caches: + continue + + offset = instr.offset + for name, size, data in (instr.cache_info or ()): + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + + yield Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, + False, None, None, instr.positions) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 772dbd1d021305..46a51007f9644d 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -670,9 +670,11 @@ def basics(): r""" 30 test.doctest_lineno.ClassWithDoctest None test.doctest_lineno.ClassWithoutDocstring None test.doctest_lineno.MethodWrapper + 53 test.doctest_lineno.MethodWrapper.classmethod_with_doctest 39 test.doctest_lineno.MethodWrapper.method_with_docstring 45 test.doctest_lineno.MethodWrapper.method_with_doctest None test.doctest_lineno.MethodWrapper.method_without_docstring + 61 test.doctest_lineno.MethodWrapper.property_with_doctest 4 test.doctest_lineno.func_with_docstring 12 test.doctest_lineno.func_with_doctest None test.doctest_lineno.func_without_docstring @@ -2922,6 +2924,9 @@ def test_unicode(): """ Traceback (most recent call last): File ... exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in raise Exception('clé') Exception: clé diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 512464f87162cd..39d4ace8d4a1d8 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -16,6 +16,7 @@ import email import email.policy +import email.utils from email.charset import Charset from email.generator import Generator, DecodedGenerator, BytesGenerator @@ -3337,15 +3338,137 @@ def test_getaddresses_comma_in_name(self): ], ) + def test_parsing_errors(self): + """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056""" + alice = 'alice@example.org' + bob = 'bob@example.com' + empty = ('', '') + + # Test utils.getaddresses() and utils.parseaddr() on malformed email + # addresses: default behavior (strict=True) rejects malformed address, + # and strict=False which tolerates malformed address. + for invalid_separator, expected_non_strict in ( + ('(', [(f'<{bob}>', alice)]), + (')', [('', alice), empty, ('', bob)]), + ('<', [('', alice), empty, ('', bob), empty]), + ('>', [('', alice), empty, ('', bob)]), + ('[', [('', f'{alice}[<{bob}>]')]), + (']', [('', alice), empty, ('', bob)]), + ('@', [empty, empty, ('', bob)]), + (';', [('', alice), empty, ('', bob)]), + (':', [('', alice), ('', bob)]), + ('.', [('', alice + '.'), ('', bob)]), + ('"', [('', alice), ('', f'<{bob}>')]), + ): + address = f'{alice}{invalid_separator}<{bob}>' + with self.subTest(address=address): + self.assertEqual(utils.getaddresses([address]), + [empty]) + self.assertEqual(utils.getaddresses([address], strict=False), + expected_non_strict) + + self.assertEqual(utils.parseaddr([address]), + empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Comma (',') is treated differently depending on strict parameter. + # Comma without quotes. + address = f'{alice},<{bob}>' + self.assertEqual(utils.getaddresses([address]), + [('', alice), ('', bob)]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('', alice), ('', bob)]) + self.assertEqual(utils.parseaddr([address]), + empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Real name between quotes containing comma. + address = '"Alice, alice@example.org" ' + expected_strict = ('Alice, alice@example.org', 'bob@example.com') + self.assertEqual(utils.getaddresses([address]), [expected_strict]) + self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) + self.assertEqual(utils.parseaddr([address]), expected_strict) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Valid parenthesis in comments. + address = 'alice@example.org (Alice)' + expected_strict = ('Alice', 'alice@example.org') + self.assertEqual(utils.getaddresses([address]), [expected_strict]) + self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict]) + self.assertEqual(utils.parseaddr([address]), expected_strict) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Invalid parenthesis in comments. + address = 'alice@example.org )Alice(' + self.assertEqual(utils.getaddresses([address]), [empty]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('', 'alice@example.org'), ('', ''), ('', 'Alice')]) + self.assertEqual(utils.parseaddr([address]), empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Two addresses with quotes separated by comma. + address = '"Jane Doe" , "John Doe" ' + self.assertEqual(utils.getaddresses([address]), + [('Jane Doe', 'jane@example.net'), + ('John Doe', 'john@example.net')]) + self.assertEqual(utils.getaddresses([address], strict=False), + [('Jane Doe', 'jane@example.net'), + ('John Doe', 'john@example.net')]) + self.assertEqual(utils.parseaddr([address]), empty) + self.assertEqual(utils.parseaddr([address], strict=False), + ('', address)) + + # Test email.utils.supports_strict_parsing attribute + self.assertEqual(email.utils.supports_strict_parsing, True) + def test_getaddresses_nasty(self): - eq = self.assertEqual - eq(utils.getaddresses(['foo: ;']), [('', '')]) - eq(utils.getaddresses( - ['[]*-- =~$']), - [('', ''), ('', ''), ('', '*--')]) - eq(utils.getaddresses( - ['foo: ;', '"Jason R. Mastaler" ']), - [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]) + for addresses, expected in ( + (['"Sürname, Firstname" '], + [('Sürname, Firstname', 'to@example.com')]), + + (['foo: ;'], + [('', '')]), + + (['foo: ;', '"Jason R. Mastaler" '], + [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]), + + ([r'Pete(A nice \) chap) '], + [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]), + + (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'], + [('', '')]), + + (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'], + [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]), + + (['John Doe '], + [('John Doe (comment)', 'jdoe@machine.example')]), + + (['"Mary Smith: Personal Account" '], + [('Mary Smith: Personal Account', 'smith@home.example')]), + + (['Undisclosed recipients:;'], + [('', '')]), + + ([r', "Giant; \"Big\" Box" '], + [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]), + ): + with self.subTest(addresses=addresses): + self.assertEqual(utils.getaddresses(addresses), + expected) + self.assertEqual(utils.getaddresses(addresses, strict=False), + expected) + + addresses = ['[]*-- =~$'] + self.assertEqual(utils.getaddresses(addresses), + [('', '')]) + self.assertEqual(utils.getaddresses(addresses, strict=False), + [('', ''), ('', ''), ('', '*--')]) def test_getaddresses_embedded_comment(self): """Test proper handling of a nested comment""" @@ -3536,6 +3659,54 @@ def test_mime_classes_policy_argument(self): m = cls(*constructor, policy=email.policy.default) self.assertIs(m.policy, email.policy.default) + def test_iter_escaped_chars(self): + self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')), + [(0, 'a'), + (2, '\\\\'), + (3, 'b'), + (5, '\\"'), + (6, 'c'), + (8, '\\\\'), + (9, '"'), + (10, 'd')]) + self.assertEqual(list(utils._iter_escaped_chars('a\\')), + [(0, 'a'), (1, '\\')]) + + def test_strip_quoted_realnames(self): + def check(addr, expected): + self.assertEqual(utils._strip_quoted_realnames(addr), expected) + + check('"Jane Doe" , "John Doe" ', + ' , ') + check(r'"Jane \"Doe\"." ', + ' ') + + # special cases + check(r'before"name"after', 'beforeafter') + check(r'before"name"', 'before') + check(r'b"name"', 'b') # single char + check(r'"name"after', 'after') + check(r'"name"a', 'a') # single char + check(r'"name"', '') + + # no change + for addr in ( + 'Jane Doe , John Doe ', + 'lone " quote', + ): + self.assertEqual(utils._strip_quoted_realnames(addr), addr) + + + def test_check_parenthesis(self): + addr = 'alice@example.net' + self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)')) + self.assertFalse(utils._check_parenthesis(f'{addr} )Alice(')) + self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))')) + self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)')) + + # Ignore real name between quotes + self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}')) + # Test the iterator/generators class TestIterators(TestEmailBase): diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index d3f396f02e7a72..034f7626c1fc7c 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -748,6 +748,35 @@ def test_iter_attachments_mutation(self): self.assertEqual(len(list(m.iter_attachments())), 2) self.assertEqual(m.get_payload(), orig) + get_payload_surrogate_params = { + + 'good_surrogateescape': ( + "String that can be encod\udcc3\udcabd with surrogateescape", + b'String that can be encod\xc3\xabd with surrogateescape' + ), + + 'string_with_utf8': ( + "String with utf-8 charactër", + b'String with utf-8 charact\xebr' + ), + + 'surrogate_and_utf8': ( + "String that cannot be ëncod\udcc3\udcabd with surrogateescape", + b'String that cannot be \xebncod\\udcc3\\udcabd with surrogateescape' + ), + + 'out_of_range_surrogate': ( + "String with \udfff cannot be encoded with surrogateescape", + b'String with \\udfff cannot be encoded with surrogateescape' + ), + } + + def get_payload_surrogate_as_gh_94606(self, msg, expected): + """test for GH issue 94606""" + m = self._str_msg(msg) + payload = m.get_payload(decode=True) + self.assertEqual(expected, payload) + class TestEmailMessage(TestEmailMessageBase, TestEmailBase): message = EmailMessage diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d2d6c1b61e46f0..6c60854bbd76cc 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -23,6 +23,12 @@ PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 PYMEM_ALLOCATOR_MALLOC = 3 +PYMEM_ALLOCATOR_MIMALLOC = 7 +if support.Py_GIL_DISABLED: + ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MIMALLOC +else: + ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MALLOC + Py_STATS = hasattr(sys, '_stats_on') # _PyCoreConfig_InitCompatConfig() @@ -841,7 +847,7 @@ def test_init_global_config(self): def test_init_from_config(self): preconfig = { - 'allocator': PYMEM_ALLOCATOR_MALLOC, + 'allocator': ALLOCATOR_FOR_CONFIG, 'utf8_mode': 1, } config = { @@ -908,7 +914,7 @@ def test_init_from_config(self): def test_init_compat_env(self): preconfig = { - 'allocator': PYMEM_ALLOCATOR_MALLOC, + 'allocator': ALLOCATOR_FOR_CONFIG, } config = { 'use_hash_seed': 1, @@ -942,7 +948,7 @@ def test_init_compat_env(self): def test_init_python_env(self): preconfig = { - 'allocator': PYMEM_ALLOCATOR_MALLOC, + 'allocator': ALLOCATOR_FOR_CONFIG, 'utf8_mode': 1, } config = { @@ -984,7 +990,7 @@ def test_init_env_dev_mode(self): api=API_COMPAT) def test_init_env_dev_mode_alloc(self): - preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC) + preconfig = dict(allocator=ALLOCATOR_FOR_CONFIG) config = dict(dev_mode=1, faulthandler=1, warnoptions=['default']) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index c602913ca69277..f99d4ca204b5a7 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -514,6 +514,7 @@ def test_contains_tf(self): self.assertFalse('first' in MainEnum) val = MainEnum.dupe self.assertIn(val, MainEnum) + self.assertNotIn(float('nan'), MainEnum) # class OtherEnum(Enum): one = auto() @@ -3268,6 +3269,65 @@ def __new__(cls, value): member._value_ = Base(value) return member + def test_extra_member_creation(self): + class IDEnumMeta(EnumMeta): + def __new__(metacls, cls, bases, classdict, **kwds): + # add new entries to classdict + for name in classdict.member_names: + classdict[f'{name}_DESC'] = f'-{classdict[name]}' + return super().__new__(metacls, cls, bases, classdict, **kwds) + class IDEnum(StrEnum, metaclass=IDEnumMeta): + pass + class MyEnum(IDEnum): + ID = 'id' + NAME = 'name' + self.assertEqual(list(MyEnum), [MyEnum.ID, MyEnum.NAME, MyEnum.ID_DESC, MyEnum.NAME_DESC]) + + def test_add_alias(self): + class mixin: + @property + def ORG(self): + return 'huh' + class Color(mixin, Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + Color.RED._add_alias_('ROJO') + self.assertIs(Color.RED, Color['ROJO']) + self.assertIs(Color.RED, Color.ROJO) + Color.BLUE._add_alias_('ORG') + self.assertIs(Color.BLUE, Color['ORG']) + self.assertIs(Color.BLUE, Color.ORG) + self.assertEqual(Color.RED.ORG, 'huh') + self.assertEqual(Color.GREEN.ORG, 'huh') + self.assertEqual(Color.BLUE.ORG, 'huh') + self.assertEqual(Color.ORG.ORG, 'huh') + + def test_add_value_alias_after_creation(self): + class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + Color.RED._add_value_alias_(5) + self.assertIs(Color.RED, Color(5)) + + def test_add_value_alias_during_creation(self): + class Types(Enum): + Unknown = 0, + Source = 1, 'src' + NetList = 2, 'nl' + def __new__(cls, int_value, *value_aliases): + member = object.__new__(cls) + member._value_ = int_value + for alias in value_aliases: + member._add_value_alias_(alias) + return member + self.assertIs(Types(0), Types.Unknown) + self.assertIs(Types(1), Types.Source) + self.assertIs(Types('src'), Types.Source) + self.assertIs(Types(2), Types.NetList) + self.assertIs(Types('nl'), Types.NetList) + class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" @@ -4941,12 +5001,14 @@ class CheckedColor(Enum): @bltns.property def zeroth(self): return 'zeroed %s' % self.name - self.assertTrue(_test_simple_enum(CheckedColor, SimpleColor) is None) + _test_simple_enum(CheckedColor, SimpleColor) SimpleColor.MAGENTA._value_ = 9 self.assertRaisesRegex( TypeError, "enum mismatch", _test_simple_enum, CheckedColor, SimpleColor, ) + # + # class CheckedMissing(IntFlag, boundary=KEEP): SIXTY_FOUR = 64 ONE_TWENTY_EIGHT = 128 @@ -4963,8 +5025,28 @@ class Missing: ALL = 2048 + 128 + 64 + 12 M = Missing self.assertEqual(list(CheckedMissing), [M.SIXTY_FOUR, M.ONE_TWENTY_EIGHT, M.TWENTY_FORTY_EIGHT]) - # _test_simple_enum(CheckedMissing, Missing) + # + # + class CheckedUnhashable(Enum): + ONE = dict() + TWO = set() + name = 'python' + self.assertIn(dict(), CheckedUnhashable) + self.assertIn('python', CheckedUnhashable) + self.assertEqual(CheckedUnhashable.name.value, 'python') + self.assertEqual(CheckedUnhashable.name.name, 'name') + # + @_simple_enum() + class Unhashable: + ONE = dict() + TWO = set() + name = 'python' + self.assertIn(dict(), Unhashable) + self.assertIn('python', Unhashable) + self.assertEqual(Unhashable.name.value, 'python') + self.assertEqual(Unhashable.name.name, 'name') + _test_simple_enum(Unhashable, Unhashable) class MiscTestCase(unittest.TestCase): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 8ccf08703e5389..c57488e44aecc6 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -2080,6 +2080,7 @@ def test_multiline_not_highlighted(self): """, [ ' 1 < 2 and', + ' 3 > 4', 'AssertionError', ], ), @@ -2087,7 +2088,7 @@ def test_multiline_not_highlighted(self): for source, expected in cases: with self.subTest(source): result = self.write_source(source) - self.assertEqual(result[-2:], expected) + self.assertEqual(result[-len(expected):], expected) class SyntaxErrorTests(unittest.TestCase): diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 499e3b6e656faa..84779526ce0eb0 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -849,12 +849,50 @@ def denominator(self): self.assertEqual(type(f.denominator), myint) def test_format_no_presentation_type(self): - # Triples (fraction, specification, expected_result) + # Triples (fraction, specification, expected_result). testcases = [ - (F(1, 3), '', '1/3'), - (F(-1, 3), '', '-1/3'), - (F(3), '', '3'), - (F(-3), '', '-3'), + # Explicit sign handling + (F(2, 3), '+', '+2/3'), + (F(-2, 3), '+', '-2/3'), + (F(3), '+', '+3'), + (F(-3), '+', '-3'), + (F(2, 3), ' ', ' 2/3'), + (F(-2, 3), ' ', '-2/3'), + (F(3), ' ', ' 3'), + (F(-3), ' ', '-3'), + (F(2, 3), '-', '2/3'), + (F(-2, 3), '-', '-2/3'), + (F(3), '-', '3'), + (F(-3), '-', '-3'), + # Padding + (F(0), '5', ' 0'), + (F(2, 3), '5', ' 2/3'), + (F(-2, 3), '5', ' -2/3'), + (F(2, 3), '0', '2/3'), + (F(2, 3), '1', '2/3'), + (F(2, 3), '2', '2/3'), + # Alignment + (F(2, 3), '<5', '2/3 '), + (F(2, 3), '>5', ' 2/3'), + (F(2, 3), '^5', ' 2/3 '), + (F(2, 3), '=5', ' 2/3'), + (F(-2, 3), '<5', '-2/3 '), + (F(-2, 3), '>5', ' -2/3'), + (F(-2, 3), '^5', '-2/3 '), + (F(-2, 3), '=5', '- 2/3'), + # Fill + (F(2, 3), 'X>5', 'XX2/3'), + (F(-2, 3), '.<5', '-2/3.'), + (F(-2, 3), '\n^6', '\n-2/3\n'), + # Thousands separators + (F(1234, 5679), ',', '1,234/5,679'), + (F(-1234, 5679), '_', '-1_234/5_679'), + (F(1234567), '_', '1_234_567'), + (F(-1234567), ',', '-1,234,567'), + # Alternate form forces a slash in the output + (F(123), '#', '123/1'), + (F(-123), '#', '-123/1'), + (F(0), '#', '0/1'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1218,6 +1256,10 @@ def test_invalid_formats(self): '.%', # Z instead of z for negative zero suppression 'Z.2f' + # z flag not supported for general formatting + 'z', + # zero padding not supported for general formatting + '05', ] for spec in invalid_specs: with self.subTest(spec=spec): diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index e4de2c5ede15f1..0ef45d3c670e85 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -939,6 +939,8 @@ def mycmp(x, y): self.assertRaises(TypeError, hash, k) self.assertNotIsInstance(k, collections.abc.Hashable) + @unittest.skipIf(support.MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_cmp_to_signature(self): self.assertEqual(str(Signature.from_callable(self.cmp_to_key)), '(mycmp)') @@ -1862,6 +1864,20 @@ def orig(): ... self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') + @support.skip_on_s390x + @unittest.skipIf(support.is_wasi, "WASI has limited C stack") + def test_lru_recursion(self): + + @self.module.lru_cache + def fib(n): + if n <= 1: + return n + return fib(n-1) + fib(n-2) + + if not support.Py_DEBUG: + with support.infinite_recursion(): + fib(2500) + @py_functools.lru_cache() def py_cached_func(x, y): diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 98a8fff4268746..3b2f579be684b7 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -15,9 +15,8 @@ def skip_if_different_mount_drives(): root_drive = os.path.splitroot(ROOT)[0] cwd_drive = os.path.splitroot(os.getcwd())[0] if root_drive != cwd_drive: - # generate_cases.py uses relpath() which raises ValueError if ROOT - # and the current working different have different mount drives - # (on Windows). + # May raise ValueError if ROOT and the current working + # different have different mount drives (on Windows). raise unittest.SkipTest( f"the current working directory and the Python source code " f"directory have different mount drives " @@ -28,10 +27,10 @@ def skip_if_different_mount_drives(): test_tools.skip_if_missing('cases_generator') with test_tools.imports_under_tool('cases_generator'): - import generate_cases - import analysis - import formatting - from parsing import StackEffect + from analyzer import StackItem + import parser + from stack import Stack + import tier1_generator def handle_stderr(): @@ -42,37 +41,24 @@ def handle_stderr(): class TestEffects(unittest.TestCase): def test_effect_sizes(self): - input_effects = [ - x := StackEffect("x", "", "", ""), - y := StackEffect("y", "", "", "oparg"), - z := StackEffect("z", "", "", "oparg*2"), + stack = Stack() + inputs = [ + x:= StackItem("x", None, "", "1"), + y:= StackItem("y", None, "", "oparg"), + z:= StackItem("z", None, "", "oparg*2"), ] - output_effects = [ - StackEffect("a", "", "", ""), - StackEffect("b", "", "", "oparg*4"), - StackEffect("c", "", "", ""), + outputs = [ + StackItem("x", None, "", "1"), + StackItem("b", None, "", "oparg*4"), + StackItem("c", None, "", "1"), ] - other_effects = [ - StackEffect("p", "", "", "oparg<<1"), - StackEffect("q", "", "", ""), - StackEffect("r", "", "", ""), - ] - self.assertEqual(formatting.effect_size(x), (1, "")) - self.assertEqual(formatting.effect_size(y), (0, "oparg")) - self.assertEqual(formatting.effect_size(z), (0, "oparg*2")) - - self.assertEqual( - formatting.list_effect_size(input_effects), - (1, "oparg + oparg*2"), - ) - self.assertEqual( - formatting.list_effect_size(output_effects), - (2, "oparg*4"), - ) - self.assertEqual( - formatting.list_effect_size(other_effects), - (2, "(oparg<<1)"), - ) + stack.pop(z) + stack.pop(y) + stack.pop(x) + for out in outputs: + stack.push(out) + self.assertEqual(stack.base_offset.to_c(), "-1 - oparg*2 - oparg") + self.assertEqual(stack.top_offset.to_c(), "1 - oparg*2 - oparg + oparg*4") class TestGeneratedCases(unittest.TestCase): @@ -103,18 +89,17 @@ def tearDown(self) -> None: def run_cases_test(self, input: str, expected: str): with open(self.temp_input_filename, "w+") as temp_input: - temp_input.write(analysis.BEGIN_MARKER) + temp_input.write(parser.BEGIN_MARKER) temp_input.write(input) - temp_input.write(analysis.END_MARKER) + temp_input.write(parser.END_MARKER) temp_input.flush() - a = generate_cases.Generator([self.temp_input_filename]) with handle_stderr(): - a.parse() - a.analyze() - if a.errors: - raise RuntimeError(f"Found {a.errors} errors") - a.write_instructions(self.temp_output_filename, False) + tier1_generator.generate_tier1_from_files( + [self.temp_input_filename], + self.temp_output_filename, + False + ) with open(self.temp_output_filename) as temp_output: lines = temp_output.readlines() @@ -163,7 +148,7 @@ def test_inst_one_pop(self): PyObject *value; value = stack_pointer[-1]; spam(); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } """ @@ -182,8 +167,8 @@ def test_inst_one_push(self): INSTRUCTION_STATS(OP); PyObject *res; spam(); - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } """ @@ -227,8 +212,8 @@ def test_binary_op(self): right = stack_pointer[-1]; left = stack_pointer[-2]; spam(); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } """ @@ -273,7 +258,6 @@ def test_predictions_and_eval_breaker(self): next_instr += 1; INSTRUCTION_STATS(OP1); PREDICTED(OP1); - static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); PyObject *arg; PyObject *rest; arg = stack_pointer[-1]; @@ -285,6 +269,7 @@ def test_predictions_and_eval_breaker(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); PyObject *arg; PyObject *res; arg = stack_pointer[-1]; @@ -325,6 +310,7 @@ def test_error_if_plain_with_comment(self): next_instr += 1; INSTRUCTION_STATS(OP); if (cond) goto label; + // Comment is ok DISPATCH(); } """ @@ -347,8 +333,8 @@ def test_error_if_pop(self): right = stack_pointer[-1]; left = stack_pointer[-2]; if (cond) goto pop_2_label; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } """ @@ -368,7 +354,7 @@ def test_cache_effect(self): value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); uint32_t extra = read_u32(&this_instr[2].cache); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } """ @@ -411,26 +397,26 @@ def test_macro_instruction(self): INSTRUCTION_STATS(OP); PREDICTED(OP); _Py_CODEUNIT *this_instr = next_instr - 6; - static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *arg2; PyObject *res; - // OP1 + // _OP1 right = stack_pointer[-1]; left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); op1(left, right); } + /* Skip 2 cache entries */ // OP2 arg2 = stack_pointer[-3]; { uint32_t extra = read_u32(&this_instr[4].cache); res = op2(arg2, left, right); } - STACK_SHRINK(2); - stack_pointer[-1] = res; + stack_pointer[-3] = res; + stack_pointer += -2; DISPATCH(); } @@ -451,16 +437,75 @@ def test_macro_instruction(self): frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *arg2; PyObject *res; + /* Skip 5 cache entries */ right = stack_pointer[-1]; left = stack_pointer[-2]; arg2 = stack_pointer[-3]; res = op3(arg2, left, right); - STACK_SHRINK(2); - stack_pointer[-1] = res; + stack_pointer[-3] = res; + stack_pointer += -2; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_caches(self): + input = """ + inst(OP, (unused/1, unused/2 --)) { + body(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + body(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_no_flags(self): + input = """ + pseudo(OP) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_with_flags(self): + input = """ + pseudo(OP, (HAS_ARG, HAS_JUMP)) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); DISPATCH(); } """ @@ -481,11 +526,10 @@ def test_array_input(self): PyObject **values; PyObject *below; above = stack_pointer[-1]; - values = stack_pointer - 1 - oparg*2; + values = &stack_pointer[-1 - oparg*2]; below = stack_pointer[-2 - oparg*2]; spam(); - STACK_SHRINK(oparg*2); - STACK_SHRINK(2); + stack_pointer += -2 - oparg*2; DISPATCH(); } """ @@ -505,11 +549,11 @@ def test_array_output(self): PyObject *below; PyObject **values; PyObject *above; - values = stack_pointer - 1; + values = &stack_pointer[-1]; spam(values, oparg); - STACK_GROW(oparg*3); - stack_pointer[-2 - oparg*3] = below; - stack_pointer[-1] = above; + stack_pointer[-2] = below; + stack_pointer[-1 + oparg*3] = above; + stack_pointer += oparg*3; DISPATCH(); } """ @@ -528,10 +572,10 @@ def test_array_input_output(self): INSTRUCTION_STATS(OP); PyObject **values; PyObject *above; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; spam(values, oparg); - STACK_GROW(1); - stack_pointer[-1] = above; + stack_pointer[0] = above; + stack_pointer += 1; DISPATCH(); } """ @@ -550,11 +594,10 @@ def test_array_error_if(self): INSTRUCTION_STATS(OP); PyObject **values; PyObject *extra; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; extra = stack_pointer[-1 - oparg]; - if (oparg == 0) { STACK_SHRINK(oparg); goto pop_1_somewhere; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); + if (oparg == 0) { stack_pointer += -1 - oparg; goto somewhere; } + stack_pointer += -1 - oparg; DISPATCH(); } """ @@ -578,14 +621,13 @@ def test_cond_effect(self): PyObject *output = NULL; PyObject *zz; cc = stack_pointer[-1]; - if ((oparg & 1) == 1) { input = stack_pointer[-1 - ((oparg & 1) == 1 ? 1 : 0)]; } - aa = stack_pointer[-2 - ((oparg & 1) == 1 ? 1 : 0)]; + if ((oparg & 1) == 1) { input = stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)]; } + aa = stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)]; output = spam(oparg, input); - STACK_SHRINK((((oparg & 1) == 1) ? 1 : 0)); - STACK_GROW(((oparg & 2) ? 1 : 0)); - stack_pointer[-2 - (oparg & 2 ? 1 : 0)] = xx; - if (oparg & 2) { stack_pointer[-1 - (oparg & 2 ? 1 : 0)] = output; } - stack_pointer[-1] = zz; + stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)] = xx; + if (oparg & 2) stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)] = output; + stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0)] = zz; + stack_pointer += -(((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0); DISPATCH(); } """ @@ -623,11 +665,10 @@ def test_macro_cond_effect(self): { # Body of B } - STACK_SHRINK(1); - STACK_GROW((oparg ? 1 : 0)); - stack_pointer[-2 - (oparg ? 1 : 0)] = deep; - if (oparg) { stack_pointer[-1 - (oparg ? 1 : 0)] = extra; } - stack_pointer[-1] = res; + stack_pointer[-3] = deep; + if (oparg) stack_pointer[-2] = extra; + stack_pointer[-2 + ((oparg) ? 1 : 0)] = res; + stack_pointer += -1 + ((oparg) ? 1 : 0); DISPATCH(); } """ @@ -658,9 +699,9 @@ def test_macro_push_push(self): { val2 = spam(); } - STACK_GROW(2); - stack_pointer[-2] = val1; - stack_pointer[-1] = val2; + stack_pointer[0] = val1; + stack_pointer[1] = val2; + stack_pointer += 2; DISPATCH(); } """ diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py index 98ecec94336e32..80dda2caaa3331 100644 --- a/Lib/test/test_getpass.py +++ b/Lib/test/test_getpass.py @@ -26,7 +26,7 @@ def test_username_priorities_of_env_values(self, environ): environ.get.return_value = None try: getpass.getuser() - except ImportError: # in case there's no pwd module + except OSError: # in case there's no pwd module pass except KeyError: # current user has no pwd entry @@ -47,7 +47,7 @@ def test_username_falls_back_to_pwd(self, environ): getpass.getuser()) getpw.assert_called_once_with(42) else: - self.assertRaises(ImportError, getpass.getuser) + self.assertRaises(OSError, getpass.getuser) class GetpassRawinputTest(unittest.TestCase): diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index caa4c76a913a01..089bf5be40a0e2 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1546,11 +1546,14 @@ def test_readline(self): resp = self.resp self._verify_readline(self.resp.readline, self.lines_expected) - def _verify_readline(self, readline, expected): + def test_readline_without_limit(self): + self._verify_readline(self.resp.readline, self.lines_expected, limit=-1) + + def _verify_readline(self, readline, expected, limit=5): all = [] while True: # short readlines - line = readline(5) + line = readline(limit) if line and line != b"foo": if len(line) < 5: self.assertTrue(line.endswith(b"\n")) @@ -1558,6 +1561,7 @@ def _verify_readline(self, readline, expected): if not line: break self.assertEqual(b"".join(all), expected) + self.assertTrue(self.resp.isclosed()) def test_read1(self): resp = self.resp @@ -1577,6 +1581,7 @@ def test_read1_unbounded(self): break all.append(data) self.assertEqual(b"".join(all), self.lines_expected) + self.assertTrue(resp.isclosed()) def test_read1_bounded(self): resp = self.resp @@ -1588,15 +1593,22 @@ def test_read1_bounded(self): self.assertLessEqual(len(data), 10) all.append(data) self.assertEqual(b"".join(all), self.lines_expected) + self.assertTrue(resp.isclosed()) def test_read1_0(self): self.assertEqual(self.resp.read1(0), b"") + self.assertFalse(self.resp.isclosed()) def test_peek_0(self): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) +class ExtendedReadTestContentLengthKnown(ExtendedReadTest): + _header, _body = ExtendedReadTest.lines.split('\r\n\r\n', 1) + lines = _header + f'\r\nContent-Length: {len(_body)}\r\n\r\n' + _body + + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 1ecac4f37fe1c1..7b0126226c4aba 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -409,9 +409,12 @@ def test_case_sensitivity(self): import RAnDoM def test_double_const(self): - # Another brief digression to test the accuracy of manifest float - # constants. - from test import double_const # don't blink -- that *was* the test + # Importing double_const checks that float constants + # serialiazed by marshal as PYC files don't lose precision + # (SF bug 422177). + from test.test_import.data import double_const + unload('test.test_import.data.double_const') + from test.test_import.data import double_const def test_import(self): def test_with_extension(ext): @@ -1629,6 +1632,14 @@ def test_circular_from_import(self): str(cm.exception), ) + def test_circular_import(self): + with self.assertRaisesRegex( + AttributeError, + r"partially initialized module 'test.test_import.data.circular_imports.import_cycle' " + r"from '.*' has no attribute 'some_attribute' \(most likely due to a circular import\)" + ): + import test.test_import.data.circular_imports.import_cycle + def test_absolute_circular_submodule(self): with self.assertRaises(AttributeError) as cm: import test.test_import.data.circular_imports.subpkg2.parent @@ -1968,6 +1979,7 @@ def test_disallowed_reimport(self): print(_testsinglephase) ''') interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) excsnap = _interpreters.run_string(interpid, script) self.assertIsNot(excsnap, None) @@ -2102,12 +2114,18 @@ def re_load(self, name, mod): def add_subinterpreter(self): interpid = _interpreters.create(isolated=False) - _interpreters.run_string(interpid, textwrap.dedent(''' + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + _interpreters.exec(interpid, textwrap.dedent(''' import sys import _testinternalcapi ''')) def clean_up(): - _interpreters.run_string(interpid, textwrap.dedent(f''' + _interpreters.exec(interpid, textwrap.dedent(f''' name = {self.NAME!r} if name in sys.modules: sys.modules.pop(name)._clear_globals() diff --git a/Lib/test/test_import/data/circular_imports/import_cycle.py b/Lib/test/test_import/data/circular_imports/import_cycle.py new file mode 100644 index 00000000000000..cd9507b5f69e25 --- /dev/null +++ b/Lib/test/test_import/data/circular_imports/import_cycle.py @@ -0,0 +1,3 @@ +import test.test_import.data.circular_imports.import_cycle as m + +m.some_attribute diff --git a/Lib/test/double_const.py b/Lib/test/test_import/data/double_const.py similarity index 100% rename from Lib/test/double_const.py rename to Lib/test/test_import/data/double_const.py diff --git a/Lib/test/test_importlib/_path.py b/Lib/test/test_importlib/_path.py index 71a704389b986e..25c799fa44cd55 100644 --- a/Lib/test/test_importlib/_path.py +++ b/Lib/test/test_importlib/_path.py @@ -1,17 +1,18 @@ -# from jaraco.path 3.5 +# from jaraco.path 3.7 import functools import pathlib -from typing import Dict, Union +from typing import Dict, Protocol, Union +from typing import runtime_checkable -try: - from typing import Protocol, runtime_checkable -except ImportError: # pragma: no cover - # Python 3.7 - from typing_extensions import Protocol, runtime_checkable # type: ignore + +class Symlink(str): + """ + A string indicating the target of a symlink. + """ -FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] # type: ignore @runtime_checkable @@ -28,6 +29,9 @@ def write_text(self, content, **kwargs): def write_bytes(self, content): ... # pragma: no cover + def symlink_to(self, target): + ... # pragma: no cover + def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore @@ -51,12 +55,16 @@ def build( ... "__init__.py": "", ... }, ... "baz.py": "# Some code", - ... } + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), ... } >>> target = getfixture('tmp_path') >>> build(spec, target) >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' """ for name, contents in spec.items(): create(contents, _ensure_tree_maker(prefix) / name) @@ -79,8 +87,8 @@ def _(content: str, path): @create.register -def _(content: str, path): - path.write_text(content, encoding='utf-8') +def _(content: Symlink, path): + path.symlink_to(content) class Recording: @@ -107,3 +115,6 @@ def write_text(self, content, **kwargs): def mkdir(self, **kwargs): return + + def symlink_to(self, target): + pass diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 64c8a5485106e3..84a0680e4ec653 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -9,6 +9,7 @@ import warnings import importlib.util import importlib +from test.support import MISSING_C_DOCSTRINGS class LoaderTests: @@ -373,7 +374,8 @@ def test_nonascii(self): with self.subTest(name): module = self.load_module_by_name(name) self.assertEqual(module.__name__, name) - self.assertEqual(module.__doc__, "Module named in %s" % lang) + if not MISSING_C_DOCSTRINGS: + self.assertEqual(module.__doc__, "Module named in %s" % lang) (Frozen_MultiPhaseExtensionModuleTests, diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py index 73e5da2ba92279..8c973356b5660d 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/fixtures.py @@ -1,6 +1,7 @@ import os import sys import copy +import json import shutil import pathlib import tempfile @@ -86,7 +87,15 @@ def setUp(self): self.fixtures.enter_context(self.add_sys_path(self.site_dir)) -class DistInfoPkg(OnSysPath, SiteDir): +class SiteBuilder(SiteDir): + def setUp(self): + super().setUp() + for cls in self.__class__.mro(): + with contextlib.suppress(AttributeError): + build_files(cls.files, prefix=self.site_dir) + + +class DistInfoPkg(OnSysPath, SiteBuilder): files: FilesSpec = { "distinfo_pkg-1.0.0.dist-info": { "METADATA": """ @@ -113,10 +122,6 @@ def main(): """, } - def setUp(self): - super().setUp() - build_files(DistInfoPkg.files, self.site_dir) - def make_uppercase(self): """ Rewrite metadata with everything uppercase. @@ -128,7 +133,28 @@ def make_uppercase(self): build_files(files, self.site_dir) -class DistInfoPkgWithDot(OnSysPath, SiteDir): +class DistInfoPkgEditable(DistInfoPkg): + """ + Package with a PEP 660 direct_url.json. + """ + + some_hash = '524127ce937f7cb65665130c695abd18ca386f60bb29687efb976faa1596fdcc' + files: FilesSpec = { + 'distinfo_pkg-1.0.0.dist-info': { + 'direct_url.json': json.dumps( + { + "archive_info": { + "hash": f"sha256={some_hash}", + "hashes": {"sha256": f"{some_hash}"}, + }, + "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", + } + ) + }, + } + + +class DistInfoPkgWithDot(OnSysPath, SiteBuilder): files: FilesSpec = { "pkg_dot-1.0.0.dist-info": { "METADATA": """ @@ -138,12 +164,8 @@ class DistInfoPkgWithDot(OnSysPath, SiteDir): }, } - def setUp(self): - super().setUp() - build_files(DistInfoPkgWithDot.files, self.site_dir) - -class DistInfoPkgWithDotLegacy(OnSysPath, SiteDir): +class DistInfoPkgWithDotLegacy(OnSysPath, SiteBuilder): files: FilesSpec = { "pkg.dot-1.0.0.dist-info": { "METADATA": """ @@ -159,18 +181,12 @@ class DistInfoPkgWithDotLegacy(OnSysPath, SiteDir): }, } - def setUp(self): - super().setUp() - build_files(DistInfoPkgWithDotLegacy.files, self.site_dir) - -class DistInfoPkgOffPath(SiteDir): - def setUp(self): - super().setUp() - build_files(DistInfoPkg.files, self.site_dir) +class DistInfoPkgOffPath(SiteBuilder): + files = DistInfoPkg.files -class EggInfoPkg(OnSysPath, SiteDir): +class EggInfoPkg(OnSysPath, SiteBuilder): files: FilesSpec = { "egginfo_pkg.egg-info": { "PKG-INFO": """ @@ -205,12 +221,8 @@ def main(): """, } - def setUp(self): - super().setUp() - build_files(EggInfoPkg.files, prefix=self.site_dir) - -class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteDir): +class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteBuilder): files: FilesSpec = { "egg_with_module_pkg.egg-info": { "PKG-INFO": "Name: egg_with_module-pkg", @@ -240,12 +252,8 @@ def main(): """, } - def setUp(self): - super().setUp() - build_files(EggInfoPkgPipInstalledNoToplevel.files, prefix=self.site_dir) - -class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteDir): +class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): files: FilesSpec = { "egg_with_no_modules_pkg.egg-info": { "PKG-INFO": "Name: egg_with_no_modules-pkg", @@ -270,12 +278,8 @@ class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteDir): }, } - def setUp(self): - super().setUp() - build_files(EggInfoPkgPipInstalledNoModules.files, prefix=self.site_dir) - -class EggInfoPkgSourcesFallback(OnSysPath, SiteDir): +class EggInfoPkgSourcesFallback(OnSysPath, SiteBuilder): files: FilesSpec = { "sources_fallback_pkg.egg-info": { "PKG-INFO": "Name: sources_fallback-pkg", @@ -296,12 +300,8 @@ def main(): """, } - def setUp(self): - super().setUp() - build_files(EggInfoPkgSourcesFallback.files, prefix=self.site_dir) - -class EggInfoFile(OnSysPath, SiteDir): +class EggInfoFile(OnSysPath, SiteBuilder): files: FilesSpec = { "egginfo_file.egg-info": """ Metadata-Version: 1.0 @@ -317,10 +317,6 @@ class EggInfoFile(OnSysPath, SiteDir): """, } - def setUp(self): - super().setUp() - build_files(EggInfoFile.files, prefix=self.site_dir) - # dedent all text strings before writing orig = _path.create.registry[str] diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py index c6996a42534676..858b37effc64bd 100644 --- a/Lib/test/test_importlib/import_/test___loader__.py +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -23,10 +23,6 @@ def test___loader__(self): with util.uncache('blah'), util.import_state(meta_path=[loader]): module = self.__import__('blah') self.assertEqual(loader, module.__loader__) - expected_repr_pattern = ( - r"\)>" - ) - self.assertRegex(repr(module), expected_repr_pattern) (Frozen_SpecTests, diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py index 3b49227255eb58..1d3817151edf64 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/test_main.py @@ -12,6 +12,7 @@ from . import fixtures from ._context import suppress +from ._path import Symlink from importlib.metadata import ( Distribution, EntryPoint, @@ -68,7 +69,7 @@ def test_abc_enforced(self): dict(name=''), ) def test_invalid_inputs_to_from_name(self, name): - with self.assertRaises(ValueError): + with self.assertRaises(Exception): Distribution.from_name(name) @@ -207,6 +208,20 @@ def test_invalid_usage(self): with self.assertRaises(ValueError): list(distributions(context='something', name='else')) + def test_interleaved_discovery(self): + """ + Ensure interleaved searches are safe. + + When the search is cached, it is possible for searches to be + interleaved, so make sure those use-cases are safe. + + Ref #293 + """ + dists = distributions() + next(dists) + version('egginfo-pkg') + next(dists) + class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): def test_egg_info(self): @@ -388,6 +403,27 @@ def test_packages_distributions_all_module_types(self): assert not any(name.endswith('.dist-info') for name in distributions) + def test_packages_distributions_symlinked_top_level(self) -> None: + """ + Distribution is resolvable from a simple top-level symlink in RECORD. + See #452. + """ + + files: fixtures.FilesSpec = { + "symlinked_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: symlinked-pkg + Version: 1.0.0 + """, + "RECORD": "symlinked,,\n", + }, + ".symlink.target": {}, + "symlinked": Symlink(".symlink.target"), + } + + fixtures.build_files(files, self.site_dir) + assert packages_distributions()['symlinked'] == ['symlinked-pkg'] + class PackagesDistributionsEggTest( fixtures.EggInfoPkg, @@ -424,3 +460,10 @@ def import_names_from_package(package_name): # sources_fallback-pkg has one import ('sources_fallback') inferred from # SOURCES.txt (top_level.txt and installed-files.txt is missing) assert import_names_from_package('sources_fallback-pkg') == {'sources_fallback'} + + +class EditableDistributionTest(fixtures.DistInfoPkgEditable, unittest.TestCase): + def test_origin(self): + dist = Distribution.from_name('distinfo-pkg') + assert dist.origin.url.endswith('.whl') + assert dist.origin.archive_info.hashes.sha256 diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 914176559806f4..fe5e7b31d9c32b 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -657,14 +657,26 @@ class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): interpid = _interpreters.create(isolated=True) - excsnap = _interpreters.run_string(interpid, script) + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) if excsnap is not None: if excsnap.type.__name__ == 'ImportError': raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): interpid = _interpreters.create(isolated=False) - excsnap = _interpreters.run_string(interpid, script) + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) if excsnap is not None: if excsnap.type.__name__ == 'ImportError': raise ImportError(excsnap.msg) diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index d25133240b1afd..8a9a8fffcd10d4 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -4,8 +4,8 @@ import os import re import sys -import sysconfig import unittest +from test import support from test.support import import_helper from contextlib import contextmanager from test.test_importlib.util import temp_module @@ -112,7 +112,7 @@ def test_module_not_found(self): class WindowsExtensionSuffixTests: def test_tagged_suffix(self): suffixes = self.machinery.EXTENSION_SUFFIXES - abi_flags = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "" + abi_flags = "t" if support.Py_GIL_DISABLED else "" ver = sys.version_info platform = re.sub('[^a-zA-Z0-9]', '_', get_platform()) expected_tag = f".cp{ver.major}{ver.minor}{abi_flags}-{platform}.pyd" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index becbb0498bbb3f..4611f62b293ff9 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -2384,6 +2384,10 @@ def test_closed_after_immediate_exception(self): self.generator.throw(RuntimeError) self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + def test_closed_after_close(self): + self.generator.close() + self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + def test_running(self): # As mentioned on issue #10220, checking for the RUNNING state only # makes sense inside the generator itself. @@ -2493,6 +2497,10 @@ def test_closed_after_immediate_exception(self): self.coroutine.throw(RuntimeError) self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + def test_closed_after_close(self): + self.coroutine.close() + self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + def test_easy_debugging(self): # repr() and str() of a coroutine state should contain the state name names = 'CORO_CREATED CORO_RUNNING CORO_SUSPENDED CORO_CLOSED'.split() @@ -3788,26 +3796,36 @@ def foo(a:int=1, *, b, c=None, **kwargs) -> 42: pass self.assertEqual(str(inspect.signature(foo)), '(a: int = 1, *, b, c=None, **kwargs) -> 42') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) def foo(a:int=1, *args, b, c=None, **kwargs) -> 42: pass self.assertEqual(str(inspect.signature(foo)), '(a: int = 1, *args, b, c=None, **kwargs) -> 42') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) def foo(): pass self.assertEqual(str(inspect.signature(foo)), '()') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) def foo(a: list[str]) -> tuple[str, float]: pass self.assertEqual(str(inspect.signature(foo)), '(a: list[str]) -> tuple[str, float]') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) from typing import Tuple def foo(a: list[str]) -> Tuple[str, float]: pass self.assertEqual(str(inspect.signature(foo)), '(a: list[str]) -> Tuple[str, float]') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) def test_signature_str_positional_only(self): P = inspect.Parameter @@ -3818,19 +3836,85 @@ def test(a_po, /, *, b, **kwargs): self.assertEqual(str(inspect.signature(test)), '(a_po, /, *, b, **kwargs)') + self.assertEqual(str(inspect.signature(test)), + inspect.signature(test).format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY)]) + self.assertEqual(str(test), '(foo, /)') + self.assertEqual(str(test), test.format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY), + P('bar', P.VAR_KEYWORD)]) + self.assertEqual(str(test), '(foo, /, **bar)') + self.assertEqual(str(test), test.format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY), + P('bar', P.VAR_POSITIONAL)]) + self.assertEqual(str(test), '(foo, /, *bar)') + self.assertEqual(str(test), test.format()) + + def test_signature_format(self): + from typing import Annotated, Literal + + def func(x: Annotated[int, 'meta'], y: Literal['a', 'b'], z: 'LiteralString'): + pass - self.assertEqual(str(S(parameters=[P('foo', P.POSITIONAL_ONLY)])), - '(foo, /)') + expected_singleline = "(x: Annotated[int, 'meta'], y: Literal['a', 'b'], z: 'LiteralString')" + expected_multiline = """( + x: Annotated[int, 'meta'], + y: Literal['a', 'b'], + z: 'LiteralString' +)""" + self.assertEqual( + inspect.signature(func).format(), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=None), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=len(expected_singleline)), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=len(expected_singleline) - 1), + expected_multiline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=0), + expected_multiline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=-1), + expected_multiline, + ) - self.assertEqual(str(S(parameters=[ - P('foo', P.POSITIONAL_ONLY), - P('bar', P.VAR_KEYWORD)])), - '(foo, /, **bar)') + def test_signature_format_all_arg_types(self): + from typing import Annotated, Literal + + def func( + x: Annotated[int, 'meta'], + /, + y: Literal['a', 'b'], + *, + z: 'LiteralString', + **kwargs: object, + ) -> None: + pass - self.assertEqual(str(S(parameters=[ - P('foo', P.POSITIONAL_ONLY), - P('bar', P.VAR_POSITIONAL)])), - '(foo, /, *bar)') + expected_multiline = """( + x: Annotated[int, 'meta'], + /, + y: Literal['a', 'b'], + *, + z: 'LiteralString', + **kwargs: object +) -> None""" + self.assertEqual( + inspect.signature(func).format(max_width=-1), + expected_multiline, + ) def test_signature_replace_parameters(self): def test(a, b) -> 42: @@ -3906,6 +3990,8 @@ def foo(a, *, b:1): pass foo_sig = MySignature.from_callable(foo) self.assertIsInstance(foo_sig, MySignature) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_signature_from_callable_class(self): # A regression test for a class inheriting its signature from `object`. class MySignature(inspect.Signature): pass @@ -3996,7 +4082,8 @@ def test_signature_eval_str(self): par('c', PORK, annotation="'MyClass'"), ))) - self.assertEqual(signature_func(isa.UnannotatedClass), sig()) + if not MISSING_C_DOCSTRINGS: + self.assertEqual(signature_func(isa.UnannotatedClass), sig()) self.assertEqual(signature_func(isa.unannotated_function), sig( parameters=( diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py deleted file mode 100644 index 5663706c0ccfb7..00000000000000 --- a/Lib/test/test_interpreters.py +++ /dev/null @@ -1,1136 +0,0 @@ -import contextlib -import json -import os -import os.path -import sys -import threading -from textwrap import dedent -import unittest -import time - -from test import support -from test.support import import_helper -from test.support import threading_helper -from test.support import os_helper -_interpreters = import_helper.import_module('_xxsubinterpreters') -_channels = import_helper.import_module('_xxinterpchannels') -from test.support import interpreters - - -def _captured_script(script): - r, w = os.pipe() - indented = script.replace('\n', '\n ') - wrapped = dedent(f""" - import contextlib - with open({w}, 'w', encoding='utf-8') as spipe: - with contextlib.redirect_stdout(spipe): - {indented} - """) - return wrapped, open(r, encoding='utf-8') - - -def clean_up_interpreters(): - for interp in interpreters.list_all(): - if interp.id == 0: # main - continue - try: - interp.close() - except RuntimeError: - pass # already destroyed - - -def _run_output(interp, request, channels=None): - script, rpipe = _captured_script(request) - with rpipe: - interp.run(script, channels=channels) - return rpipe.read() - - -@contextlib.contextmanager -def _running(interp): - r, w = os.pipe() - def run(): - interp.run(dedent(f""" - # wait for "signal" - with open({r}) as rpipe: - rpipe.read() - """)) - - t = threading.Thread(target=run) - t.start() - - yield - - with open(w, 'w') as spipe: - spipe.write('done') - t.join() - - -class TestBase(unittest.TestCase): - - def pipe(self): - def ensure_closed(fd): - try: - os.close(fd) - except OSError: - pass - r, w = os.pipe() - self.addCleanup(lambda: ensure_closed(r)) - self.addCleanup(lambda: ensure_closed(w)) - return r, w - - def tearDown(self): - clean_up_interpreters() - - -class CreateTests(TestBase): - - def test_in_main(self): - interp = interpreters.create() - self.assertIsInstance(interp, interpreters.Interpreter) - self.assertIn(interp, interpreters.list_all()) - - def test_in_thread(self): - lock = threading.Lock() - interp = None - def f(): - nonlocal interp - interp = interpreters.create() - lock.acquire() - lock.release() - t = threading.Thread(target=f) - with lock: - t.start() - t.join() - self.assertIn(interp, interpreters.list_all()) - - def test_in_subinterpreter(self): - main, = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - interp = interpreters.create() - print(interp.id) - """)) - interp2 = interpreters.Interpreter(int(out)) - self.assertEqual(interpreters.list_all(), [main, interp, interp2]) - - def test_after_destroy_all(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - interp_lst = [] - for _ in range(3): - interps = interpreters.create() - interp_lst.append(interps) - # Now destroy them. - for interp in interp_lst: - interp.close() - # Finally, create another. - interp = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {interp}) - - def test_after_destroy_some(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - # Now destroy 2 of them. - interp1.close() - interp2.close() - # Finally, create another. - interp = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) - - -class GetCurrentTests(TestBase): - - def test_main(self): - main = interpreters.get_main() - current = interpreters.get_current() - self.assertEqual(current, main) - - def test_subinterpreter(self): - main = _interpreters.get_main() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - cur = interpreters.get_current() - print(cur.id) - """)) - current = interpreters.Interpreter(int(out)) - self.assertNotEqual(current, main) - - -class ListAllTests(TestBase): - - def test_initial(self): - interps = interpreters.list_all() - self.assertEqual(1, len(interps)) - - def test_after_creating(self): - main = interpreters.get_current() - first = interpreters.create() - second = interpreters.create() - - ids = [] - for interp in interpreters.list_all(): - ids.append(interp.id) - - self.assertEqual(ids, [main.id, first.id, second.id]) - - def test_after_destroying(self): - main = interpreters.get_current() - first = interpreters.create() - second = interpreters.create() - first.close() - - ids = [] - for interp in interpreters.list_all(): - ids.append(interp.id) - - self.assertEqual(ids, [main.id, second.id]) - - -class TestInterpreterAttrs(TestBase): - - def test_id_type(self): - main = interpreters.get_main() - current = interpreters.get_current() - interp = interpreters.create() - self.assertIsInstance(main.id, _interpreters.InterpreterID) - self.assertIsInstance(current.id, _interpreters.InterpreterID) - self.assertIsInstance(interp.id, _interpreters.InterpreterID) - - def test_main_id(self): - main = interpreters.get_main() - self.assertEqual(main.id, 0) - - def test_custom_id(self): - interp = interpreters.Interpreter(1) - self.assertEqual(interp.id, 1) - - with self.assertRaises(TypeError): - interpreters.Interpreter('1') - - def test_id_readonly(self): - interp = interpreters.Interpreter(1) - with self.assertRaises(AttributeError): - interp.id = 2 - - @unittest.skip('not ready yet (see bpo-32604)') - def test_main_isolated(self): - main = interpreters.get_main() - self.assertFalse(main.isolated) - - @unittest.skip('not ready yet (see bpo-32604)') - def test_subinterpreter_isolated_default(self): - interp = interpreters.create() - self.assertFalse(interp.isolated) - - def test_subinterpreter_isolated_explicit(self): - interp1 = interpreters.create(isolated=True) - interp2 = interpreters.create(isolated=False) - self.assertTrue(interp1.isolated) - self.assertFalse(interp2.isolated) - - @unittest.skip('not ready yet (see bpo-32604)') - def test_custom_isolated_default(self): - interp = interpreters.Interpreter(1) - self.assertFalse(interp.isolated) - - def test_custom_isolated_explicit(self): - interp1 = interpreters.Interpreter(1, isolated=True) - interp2 = interpreters.Interpreter(1, isolated=False) - self.assertTrue(interp1.isolated) - self.assertFalse(interp2.isolated) - - def test_isolated_readonly(self): - interp = interpreters.Interpreter(1) - with self.assertRaises(AttributeError): - interp.isolated = True - - def test_equality(self): - interp1 = interpreters.create() - interp2 = interpreters.create() - self.assertEqual(interp1, interp1) - self.assertNotEqual(interp1, interp2) - - -class TestInterpreterIsRunning(TestBase): - - def test_main(self): - main = interpreters.get_main() - self.assertTrue(main.is_running()) - - @unittest.skip('Fails on FreeBSD') - def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interp.is_running()) - - with _running(interp): - self.assertTrue(interp.is_running()) - self.assertFalse(interp.is_running()) - - def test_finished(self): - r, w = self.pipe() - interp = interpreters.create() - interp.run(f"""if True: - import os - os.write({w}, b'x') - """) - self.assertFalse(interp.is_running()) - self.assertEqual(os.read(r, 1), b'x') - - def test_from_subinterpreter(self): - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - if _interpreters.is_running({interp.id}): - print(True) - else: - print(False) - """)) - self.assertEqual(out.strip(), 'True') - - def test_already_destroyed(self): - interp = interpreters.create() - interp.close() - with self.assertRaises(RuntimeError): - interp.is_running() - - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.is_running() - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.is_running() - - def test_with_only_background_threads(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - DONE = b'D' - FINISHED = b'F' - - interp = interpreters.create() - interp.run(f"""if True: - import os - import threading - - def task(): - v = os.read({r_thread}, 1) - assert v == {DONE!r} - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - """) - self.assertFalse(interp.is_running()) - - os.write(w_thread, DONE) - interp.run('t.join()') - self.assertEqual(os.read(r_interp, 1), FINISHED) - - -class TestInterpreterClose(TestBase): - - def test_basic(self): - main = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp2, interp3}) - interp2.close() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp3}) - - def test_all(self): - before = set(interpreters.list_all()) - interps = set() - for _ in range(3): - interp = interpreters.create() - interps.add(interp) - self.assertEqual(set(interpreters.list_all()), before | interps) - for interp in interps: - interp.close() - self.assertEqual(set(interpreters.list_all()), before) - - def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(RuntimeError): - main.close() - - def f(): - with self.assertRaises(RuntimeError): - main.close() - - t = threading.Thread(target=f) - t.start() - t.join() - - def test_already_destroyed(self): - interp = interpreters.create() - interp.close() - with self.assertRaises(RuntimeError): - interp.close() - - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.close() - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.close() - - def test_from_current(self): - main, = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - from test.support import interpreters - interp = interpreters.Interpreter({int(interp.id)}) - try: - interp.close() - except RuntimeError: - print('failed') - """)) - self.assertEqual(out.strip(), 'failed') - self.assertEqual(set(interpreters.list_all()), {main, interp}) - - def test_from_sibling(self): - main, = interpreters.list_all() - interp1 = interpreters.create() - interp2 = interpreters.create() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp2}) - interp1.run(dedent(f""" - from test.support import interpreters - interp2 = interpreters.Interpreter(int({interp2.id})) - interp2.close() - interp3 = interpreters.create() - interp3.close() - """)) - self.assertEqual(set(interpreters.list_all()), {main, interp1}) - - def test_from_other_thread(self): - interp = interpreters.create() - def f(): - interp.close() - - t = threading.Thread(target=f) - t.start() - t.join() - - @unittest.skip('Fails on FreeBSD') - def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interp.close() - self.assertTrue(interp.is_running()) - - def test_subthreads_still_running(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - FINISHED = b'F' - - interp = interpreters.create() - interp.run(f"""if True: - import os - import threading - import time - - done = False - - def notify_fini(): - global done - done = True - t.join() - threading._register_atexit(notify_fini) - - def task(): - while not done: - time.sleep(0.1) - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - """) - interp.close() - - self.assertEqual(os.read(r_interp, 1), FINISHED) - - -class TestInterpreterRun(TestBase): - - def test_success(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - interp.run(script) - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_failure(self): - interp = interpreters.create() - with self.assertRaises(interpreters.RunFailedError): - interp.run('raise Exception') - - def test_in_thread(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - def f(): - interp.run(script) - - t = threading.Thread(target=f) - t.start() - t.join() - out = file.read() - - self.assertEqual(out, 'it worked!') - - @support.requires_fork() - def test_fork(self): - interp = interpreters.create() - import tempfile - with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: - file.write('') - file.flush() - - expected = 'spam spam spam spam spam' - script = dedent(f""" - import os - try: - os.fork() - except RuntimeError: - with open('{file.name}', 'w', encoding='utf-8') as out: - out.write('{expected}') - """) - interp.run(script) - - file.seek(0) - content = file.read() - self.assertEqual(content, expected) - - @unittest.skip('Fails on FreeBSD') - def test_already_running(self): - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interp.run('print("spam")') - - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.run('print("spam")') - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.run('print("spam")') - - def test_bad_script(self): - interp = interpreters.create() - with self.assertRaises(TypeError): - interp.run(10) - - def test_bytes_for_script(self): - interp = interpreters.create() - with self.assertRaises(TypeError): - interp.run(b'print("spam")') - - def test_with_background_threads_still_running(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - RAN = b'R' - DONE = b'D' - FINISHED = b'F' - - interp = interpreters.create() - interp.run(f"""if True: - import os - import threading - - def task(): - v = os.read({r_thread}, 1) - assert v == {DONE!r} - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - os.write({w_interp}, {RAN!r}) - """) - interp.run(f"""if True: - os.write({w_interp}, {RAN!r}) - """) - - os.write(w_thread, DONE) - interp.run('t.join()') - self.assertEqual(os.read(r_interp, 1), RAN) - self.assertEqual(os.read(r_interp, 1), RAN) - self.assertEqual(os.read(r_interp, 1), FINISHED) - - # test_xxsubinterpreters covers the remaining Interpreter.run() behavior. - - -class StressTests(TestBase): - - # In these tests we generally want a lot of interpreters, - # but not so many that any test takes too long. - - @support.requires_resource('cpu') - def test_create_many_sequential(self): - alive = [] - for _ in range(100): - interp = interpreters.create() - alive.append(interp) - - @support.requires_resource('cpu') - def test_create_many_threaded(self): - alive = [] - def task(): - interp = interpreters.create() - alive.append(interp) - threads = (threading.Thread(target=task) for _ in range(200)) - with threading_helper.start_threads(threads): - pass - - -class StartupTests(TestBase): - - # We want to ensure the initial state of subinterpreters - # matches expectations. - - _subtest_count = 0 - - @contextlib.contextmanager - def subTest(self, *args): - with super().subTest(*args) as ctx: - self._subtest_count += 1 - try: - yield ctx - finally: - if self._debugged_in_subtest: - if self._subtest_count == 1: - # The first subtest adds a leading newline, so we - # compensate here by not printing a trailing newline. - print('### end subtest debug ###', end='') - else: - print('### end subtest debug ###') - self._debugged_in_subtest = False - - def debug(self, msg, *, header=None): - if header: - self._debug(f'--- {header} ---') - if msg: - if msg.endswith(os.linesep): - self._debug(msg[:-len(os.linesep)]) - else: - self._debug(msg) - self._debug('') - self._debug('------') - else: - self._debug(msg) - - _debugged = False - _debugged_in_subtest = False - def _debug(self, msg): - if not self._debugged: - print() - self._debugged = True - if self._subtest is not None: - if True: - if not self._debugged_in_subtest: - self._debugged_in_subtest = True - print('### start subtest debug ###') - print(msg) - else: - print(msg) - - def create_temp_dir(self): - import tempfile - tmp = tempfile.mkdtemp(prefix='test_interpreters_') - tmp = os.path.realpath(tmp) - self.addCleanup(os_helper.rmtree, tmp) - return tmp - - def write_script(self, *path, text): - filename = os.path.join(*path) - dirname = os.path.dirname(filename) - if dirname: - os.makedirs(dirname, exist_ok=True) - with open(filename, 'w', encoding='utf-8') as outfile: - outfile.write(dedent(text)) - return filename - - @support.requires_subprocess() - def run_python(self, argv, *, cwd=None): - # This method is inspired by - # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. - import shlex - import subprocess - if isinstance(argv, str): - argv = shlex.split(argv) - argv = [sys.executable, *argv] - try: - proc = subprocess.run( - argv, - cwd=cwd, - capture_output=True, - text=True, - ) - except Exception as exc: - self.debug(f'# cmd: {shlex.join(argv)}') - if isinstance(exc, FileNotFoundError) and not exc.filename: - if os.path.exists(argv[0]): - exists = 'exists' - else: - exists = 'does not exist' - self.debug(f'{argv[0]} {exists}') - raise # re-raise - assert proc.stderr == '' or proc.returncode != 0, proc.stderr - if proc.returncode != 0 and support.verbose: - self.debug(f'# python3 {shlex.join(argv[1:])} failed:') - self.debug(proc.stdout, header='stdout') - self.debug(proc.stderr, header='stderr') - self.assertEqual(proc.returncode, 0) - self.assertEqual(proc.stderr, '') - return proc.stdout - - def test_sys_path_0(self): - # The main interpreter's sys.path[0] should be used by subinterpreters. - script = ''' - import sys - from test.support import interpreters - - orig = sys.path[0] - - interp = interpreters.create() - interp.run(f"""if True: - import json - import sys - print(json.dumps({{ - 'main': {orig!r}, - 'sub': sys.path[0], - }}, indent=4), flush=True) - """) - ''' - # / - # pkg/ - # __init__.py - # __main__.py - # script.py - # script.py - cwd = self.create_temp_dir() - self.write_script(cwd, 'pkg', '__init__.py', text='') - self.write_script(cwd, 'pkg', '__main__.py', text=script) - self.write_script(cwd, 'pkg', 'script.py', text=script) - self.write_script(cwd, 'script.py', text=script) - - cases = [ - ('script.py', cwd), - ('-m script', cwd), - ('-m pkg', cwd), - ('-m pkg.script', cwd), - ('-c "import script"', ''), - ] - for argv, expected in cases: - with self.subTest(f'python3 {argv}'): - out = self.run_python(argv, cwd=cwd) - data = json.loads(out) - sp0_main, sp0_sub = data['main'], data['sub'] - self.assertEqual(sp0_sub, sp0_main) - self.assertEqual(sp0_sub, expected) - # XXX Also check them all with the -P cmdline flag? - - -class FinalizationTests(TestBase): - - def test_gh_109793(self): - import subprocess - argv = [sys.executable, '-c', '''if True: - import _xxsubinterpreters as _interpreters - interpid = _interpreters.create() - raise Exception - '''] - proc = subprocess.run(argv, capture_output=True, text=True) - self.assertIn('Traceback', proc.stderr) - if proc.returncode == 0 and support.verbose: - print() - print("--- cmd unexpected succeeded ---") - print(f"stdout:\n{proc.stdout}") - print(f"stderr:\n{proc.stderr}") - print("------") - self.assertEqual(proc.returncode, 1) - - -class TestIsShareable(TestBase): - - def test_default_shareables(self): - shareables = [ - # singletons - None, - # builtin objects - b'spam', - 'spam', - 10, - -10, - True, - False, - 100.0, - (), - (1, ('spam', 'eggs'), True), - ] - for obj in shareables: - with self.subTest(obj): - shareable = interpreters.is_shareable(obj) - self.assertTrue(shareable) - - def test_not_shareable(self): - class Cheese: - def __init__(self, name): - self.name = name - def __str__(self): - return self.name - - class SubBytes(bytes): - """A subclass of a shareable type.""" - - not_shareables = [ - # singletons - NotImplemented, - ..., - # builtin types and objects - type, - object, - object(), - Exception(), - # user-defined types and objects - Cheese, - Cheese('Wensleydale'), - SubBytes(b'spam'), - ] - for obj in not_shareables: - with self.subTest(repr(obj)): - self.assertFalse( - interpreters.is_shareable(obj)) - - -class TestChannels(TestBase): - - def test_create(self): - r, s = interpreters.create_channel() - self.assertIsInstance(r, interpreters.RecvChannel) - self.assertIsInstance(s, interpreters.SendChannel) - - def test_list_all(self): - self.assertEqual(interpreters.list_all_channels(), []) - created = set() - for _ in range(3): - ch = interpreters.create_channel() - created.add(ch) - after = set(interpreters.list_all_channels()) - self.assertEqual(after, created) - - def test_shareable(self): - rch, sch = interpreters.create_channel() - - self.assertTrue( - interpreters.is_shareable(rch)) - self.assertTrue( - interpreters.is_shareable(sch)) - - sch.send_nowait(rch) - sch.send_nowait(sch) - rch2 = rch.recv() - sch2 = rch.recv() - - self.assertEqual(rch2, rch) - self.assertEqual(sch2, sch) - - def test_is_closed(self): - rch, sch = interpreters.create_channel() - rbefore = rch.is_closed - sbefore = sch.is_closed - rch.close() - rafter = rch.is_closed - safter = sch.is_closed - - self.assertFalse(rbefore) - self.assertFalse(sbefore) - self.assertTrue(rafter) - self.assertTrue(safter) - - -class TestRecvChannelAttrs(TestBase): - - def test_id_type(self): - rch, _ = interpreters.create_channel() - self.assertIsInstance(rch.id, _channels.ChannelID) - - def test_custom_id(self): - rch = interpreters.RecvChannel(1) - self.assertEqual(rch.id, 1) - - with self.assertRaises(TypeError): - interpreters.RecvChannel('1') - - def test_id_readonly(self): - rch = interpreters.RecvChannel(1) - with self.assertRaises(AttributeError): - rch.id = 2 - - def test_equality(self): - ch1, _ = interpreters.create_channel() - ch2, _ = interpreters.create_channel() - self.assertEqual(ch1, ch1) - self.assertNotEqual(ch1, ch2) - - -class TestSendChannelAttrs(TestBase): - - def test_id_type(self): - _, sch = interpreters.create_channel() - self.assertIsInstance(sch.id, _channels.ChannelID) - - def test_custom_id(self): - sch = interpreters.SendChannel(1) - self.assertEqual(sch.id, 1) - - with self.assertRaises(TypeError): - interpreters.SendChannel('1') - - def test_id_readonly(self): - sch = interpreters.SendChannel(1) - with self.assertRaises(AttributeError): - sch.id = 2 - - def test_equality(self): - _, ch1 = interpreters.create_channel() - _, ch2 = interpreters.create_channel() - self.assertEqual(ch1, ch1) - self.assertNotEqual(ch1, ch2) - - -class TestSendRecv(TestBase): - - def test_send_recv_main(self): - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_same_interpreter(self): - interp = interpreters.create() - interp.run(dedent(""" - from test.support import interpreters - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv() - assert obj == orig, 'expected: obj == orig' - assert obj is not orig, 'expected: obj is not orig' - """)) - - @unittest.skip('broken (see BPO-...)') - def test_send_recv_different_interpreters(self): - r1, s1 = interpreters.create_channel() - r2, s2 = interpreters.create_channel() - orig1 = b'spam' - s1.send_nowait(orig1) - out = _run_output( - interpreters.create(), - dedent(f""" - obj1 = r.recv() - assert obj1 == b'spam', 'expected: obj1 == orig1' - # When going to another interpreter we get a copy. - assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' - orig2 = b'eggs' - print(id(orig2)) - s.send_nowait(orig2) - """), - channels=dict(r=r1, s=s2), - ) - obj2 = r2.recv() - - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) - - def test_send_recv_different_threads(self): - r, s = interpreters.create_channel() - - def f(): - while True: - try: - obj = r.recv() - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - s.send(obj) - t = threading.Thread(target=f) - t.start() - - orig = b'spam' - s.send(orig) - obj = r.recv() - t.join() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_nowait_main(self): - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv_nowait() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_nowait_main_with_default(self): - r, _ = interpreters.create_channel() - obj = r.recv_nowait(None) - - self.assertIsNone(obj) - - def test_send_recv_nowait_same_interpreter(self): - interp = interpreters.create() - interp.run(dedent(""" - from test.support import interpreters - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv_nowait() - assert obj == orig, 'expected: obj == orig' - # When going back to the same interpreter we get the same object. - assert obj is not orig, 'expected: obj is not orig' - """)) - - @unittest.skip('broken (see BPO-...)') - def test_send_recv_nowait_different_interpreters(self): - r1, s1 = interpreters.create_channel() - r2, s2 = interpreters.create_channel() - orig1 = b'spam' - s1.send_nowait(orig1) - out = _run_output( - interpreters.create(), - dedent(f""" - obj1 = r.recv_nowait() - assert obj1 == b'spam', 'expected: obj1 == orig1' - # When going to another interpreter we get a copy. - assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' - orig2 = b'eggs' - print(id(orig2)) - s.send_nowait(orig2) - """), - channels=dict(r=r1, s=s2), - ) - obj2 = r2.recv_nowait() - - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) - - def test_recv_timeout(self): - r, _ = interpreters.create_channel() - with self.assertRaises(TimeoutError): - r.recv(timeout=1) - - def test_recv_channel_does_not_exist(self): - ch = interpreters.RecvChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.recv() - - def test_send_channel_does_not_exist(self): - ch = interpreters.SendChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.send(b'spam') - - def test_recv_nowait_channel_does_not_exist(self): - ch = interpreters.RecvChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.recv_nowait() - - def test_send_nowait_channel_does_not_exist(self): - ch = interpreters.SendChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.send_nowait(b'spam') - - def test_recv_nowait_empty(self): - ch, _ = interpreters.create_channel() - with self.assertRaises(interpreters.ChannelEmptyError): - ch.recv_nowait() - - def test_recv_nowait_default(self): - default = object() - rch, sch = interpreters.create_channel() - obj1 = rch.recv_nowait(default) - sch.send_nowait(None) - sch.send_nowait(1) - sch.send_nowait(b'spam') - sch.send_nowait(b'eggs') - obj2 = rch.recv_nowait(default) - obj3 = rch.recv_nowait(default) - obj4 = rch.recv_nowait() - obj5 = rch.recv_nowait(default) - obj6 = rch.recv_nowait(default) - - self.assertIs(obj1, default) - self.assertIs(obj2, None) - self.assertEqual(obj3, 1) - self.assertEqual(obj4, b'spam') - self.assertEqual(obj5, b'eggs') - self.assertIs(obj6, default) - - def test_send_buffer(self): - buf = bytearray(b'spamspamspam') - obj = None - rch, sch = interpreters.create_channel() - - def f(): - nonlocal obj - while True: - try: - obj = rch.recv() - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - t = threading.Thread(target=f) - t.start() - - sch.send_buffer(buf) - t.join() - - self.assertIsNot(obj, buf) - self.assertIsInstance(obj, memoryview) - self.assertEqual(obj, buf) - - buf[4:8] = b'eggs' - self.assertEqual(obj, buf) - obj[4:8] = b'ham.' - self.assertEqual(obj, buf) - - def test_send_buffer_nowait(self): - buf = bytearray(b'spamspamspam') - rch, sch = interpreters.create_channel() - sch.send_buffer_nowait(buf) - obj = rch.recv() - - self.assertIsNot(obj, buf) - self.assertIsInstance(obj, memoryview) - self.assertEqual(obj, buf) - - buf[4:8] = b'eggs' - self.assertEqual(obj, buf) - obj[4:8] = b'ham.' - self.assertEqual(obj, buf) diff --git a/Lib/test/test_interpreters/__init__.py b/Lib/test/test_interpreters/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_interpreters/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_interpreters/__main__.py b/Lib/test/test_interpreters/__main__.py new file mode 100644 index 00000000000000..40a23a297ec2b4 --- /dev/null +++ b/Lib/test/test_interpreters/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py new file mode 100644 index 00000000000000..aefd326977095f --- /dev/null +++ b/Lib/test/test_interpreters/test_api.py @@ -0,0 +1,747 @@ +import os +import threading +from textwrap import dedent +import unittest + +from test import support +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxsubinterpreters') +from test.support import interpreters +from test.support.interpreters import InterpreterNotFoundError +from .utils import _captured_script, _run_output, _running, TestBase + + +class ModuleTests(TestBase): + + def test_queue_aliases(self): + first = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + second = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + self.assertEqual(second, first) + + +class CreateTests(TestBase): + + def test_in_main(self): + interp = interpreters.create() + self.assertIsInstance(interp, interpreters.Interpreter) + self.assertIn(interp, interpreters.list_all()) + + def test_in_thread(self): + lock = threading.Lock() + interp = None + def f(): + nonlocal interp + interp = interpreters.create() + lock.acquire() + lock.release() + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(interp, interpreters.list_all()) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + interp = interpreters.create() + print(interp.id) + """)) + interp2 = interpreters.Interpreter(int(out)) + self.assertEqual(interpreters.list_all(), [main, interp, interp2]) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp_lst = [] + for _ in range(3): + interps = interpreters.create() + interp_lst.append(interps) + # Now destroy them. + for interp in interp_lst: + interp.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp}) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + # Now destroy 2 of them. + interp1.close() + interp2.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) + + +class GetMainTests(TestBase): + + def test_id(self): + main = interpreters.get_main() + self.assertEqual(main.id, 0) + + def test_current(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertIs(main, current) + + def test_idempotent(self): + main1 = interpreters.get_main() + main2 = interpreters.get_main() + self.assertIs(main1, main2) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertEqual(current, main) + + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(cur.id) + """)) + current = interpreters.Interpreter(int(out)) + self.assertEqual(current, interp) + self.assertNotEqual(current, main) + + def test_idempotent(self): + with self.subTest('main'): + cur1 = interpreters.get_current() + cur2 = interpreters.get_current() + self.assertIs(cur1, cur2) + + with self.subTest('subinterpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + cur = interpreters.get_current() + print(id(cur)) + """)) + objid1, objid2 = (int(v) for v in out.splitlines()) + self.assertEqual(objid1, objid2) + + with self.subTest('per-interpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + """)) + id1 = int(out) + id2 = id(interp) + self.assertNotEqual(id1, id2) + + +class ListAllTests(TestBase): + + def test_initial(self): + interps = interpreters.list_all() + self.assertEqual(1, len(interps)) + + def test_after_creating(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, first.id, second.id]) + + def test_after_destroying(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + first.close() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, second.id]) + + def test_idempotent(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + expected = [main, first, second] + + actual = interpreters.list_all() + + self.assertEqual(actual, expected) + for interp1, interp2 in zip(actual, expected): + self.assertIs(interp1, interp2) + + +class InterpreterObjectTests(TestBase): + + def test_init_int(self): + interpid = interpreters.get_current().id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp.id, interpid) + + def test_init_interpreter_id(self): + interpid = interpreters.get_current()._id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp._id, interpid) + + def test_init_unsupported(self): + actualid = interpreters.get_current().id + for interpid in [ + str(actualid), + float(actualid), + object(), + None, + '', + ]: + with self.subTest(repr(interpid)): + with self.assertRaises(TypeError): + interpreters.Interpreter(interpid) + + def test_idempotent(self): + main = interpreters.get_main() + interp = interpreters.Interpreter(main.id) + self.assertIs(interp, main) + + def test_init_does_not_exist(self): + with self.assertRaises(InterpreterNotFoundError): + interpreters.Interpreter(1_000_000) + + def test_init_bad_id(self): + with self.assertRaises(ValueError): + interpreters.Interpreter(-1) + + def test_id_type(self): + main = interpreters.get_main() + current = interpreters.get_current() + interp = interpreters.create() + self.assertIsInstance(main.id, int) + self.assertIsInstance(current.id, int) + self.assertIsInstance(interp.id, int) + + def test_id_readonly(self): + interp = interpreters.create() + with self.assertRaises(AttributeError): + interp.id = 1_000_000 + + def test_hashable(self): + interp = interpreters.create() + expected = hash(interp.id) + actual = hash(interp) + self.assertEqual(actual, expected) + + def test_equality(self): + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(interp1, interp1) + self.assertNotEqual(interp1, interp2) + + +class TestInterpreterIsRunning(TestBase): + + def test_main(self): + main = interpreters.get_main() + self.assertTrue(main.is_running()) + + @unittest.skip('Fails on FreeBSD') + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interp.is_running()) + + with _running(interp): + self.assertTrue(interp.is_running()) + self.assertFalse(interp.is_running()) + + def test_finished(self): + r, w = self.pipe() + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + os.write({w}, b'x') + """) + self.assertFalse(interp.is_running()) + self.assertEqual(os.read(r, 1), b'x') + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + if _interpreters.is_running({interp.id}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.is_running() + + def test_with_only_background_threads(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + self.assertFalse(interp.is_running()) + + os.write(w_thread, DONE) + interp.exec_sync('t.join()') + self.assertEqual(os.read(r_interp, 1), FINISHED) + + +class TestInterpreterClose(TestBase): + + def test_basic(self): + main = interpreters.get_main() + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2, interp3}) + interp2.close() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp3}) + + def test_all(self): + before = set(interpreters.list_all()) + interps = set() + for _ in range(3): + interp = interpreters.create() + interps.add(interp) + self.assertEqual(set(interpreters.list_all()), before | interps) + for interp in interps: + interp.close() + self.assertEqual(set(interpreters.list_all()), before) + + def test_main(self): + main, = interpreters.list_all() + with self.assertRaises(RuntimeError): + main.close() + + def f(): + with self.assertRaises(RuntimeError): + main.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.close() + + def test_from_current(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + from test.support import interpreters + interp = interpreters.Interpreter({interp.id}) + try: + interp.close() + except RuntimeError: + print('failed') + """)) + self.assertEqual(out.strip(), 'failed') + self.assertEqual(set(interpreters.list_all()), {main, interp}) + + def test_from_sibling(self): + main, = interpreters.list_all() + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2}) + interp1.exec_sync(dedent(f""" + from test.support import interpreters + interp2 = interpreters.Interpreter({interp2.id}) + interp2.close() + interp3 = interpreters.create() + interp3.close() + """)) + self.assertEqual(set(interpreters.list_all()), {main, interp1}) + + def test_from_other_thread(self): + interp = interpreters.create() + def f(): + interp.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + @unittest.skip('Fails on FreeBSD') + def test_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.close() + self.assertTrue(interp.is_running()) + + def test_subthreads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + FINISHED = b'F' + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + + +class TestInterpreterPrepareMain(TestBase): + + def test_empty(self): + interp = interpreters.create() + with self.assertRaises(ValueError): + interp.prepare_main() + + def test_dict(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_tuple(self): + values = {'spam': 42, 'eggs': 'ham'} + values = tuple(values.items()) + interp = interpreters.create() + interp.prepare_main(values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_kwargs(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(**values) + out = _run_output(interp, dedent(""" + print(spam, eggs) + """)) + self.assertEqual(out.strip(), '42 ham') + + def test_dict_and_kwargs(self): + values = {'spam': 42, 'eggs': 'ham'} + interp = interpreters.create() + interp.prepare_main(values, foo='bar') + out = _run_output(interp, dedent(""" + print(spam, eggs, foo) + """)) + self.assertEqual(out.strip(), '42 ham bar') + + def test_not_shareable(self): + interp = interpreters.create() + # XXX TypeError? + with self.assertRaises(ValueError): + interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) + + # Make sure neither was actually bound. + with self.assertRaises(interpreters.ExecFailure): + interp.exec_sync('print(foo)') + with self.assertRaises(interpreters.ExecFailure): + interp.exec_sync('print(spam)') + + +class TestInterpreterExecSync(TestBase): + + def test_success(self): + interp = interpreters.create() + script, file = _captured_script('print("it worked!", end="")') + with file: + interp.exec_sync(script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_failure(self): + interp = interpreters.create() + with self.assertRaises(interpreters.ExecFailure): + interp.exec_sync('raise Exception') + + def test_display_preserved_exception(self): + tempdir = self.temp_dir() + modfile = self.make_module('spam', tempdir, text=""" + def ham(): + raise RuntimeError('uh-oh!') + + def eggs(): + ham() + """) + scriptfile = self.make_script('script.py', tempdir, text=""" + from test.support import interpreters + + def script(): + import spam + spam.eggs() + + interp = interpreters.create() + interp.exec_sync(script) + """) + + stdout, stderr = self.assert_python_failure(scriptfile) + self.maxDiff = None + interpmod_line, = (l for l in stderr.splitlines() if ' exec_sync' in l) + # File "{interpreters.__file__}", line 179, in exec_sync + self.assertEqual(stderr, dedent(f"""\ + Traceback (most recent call last): + File "{scriptfile}", line 9, in + interp.exec_sync(script) + ~~~~~~~~~~~~~~~~^^^^^^^^ + {interpmod_line.strip()} + raise ExecFailure(excinfo) + test.support.interpreters.ExecFailure: RuntimeError: uh-oh! + + Uncaught in the interpreter: + + Traceback (most recent call last): + File "{scriptfile}", line 6, in script + spam.eggs() + ~~~~~~~~~^^ + File "{modfile}", line 6, in eggs + ham() + ~~~^^ + File "{modfile}", line 3, in ham + raise RuntimeError('uh-oh!') + RuntimeError: uh-oh! + """)) + self.assertEqual(stdout, '') + + def test_in_thread(self): + interp = interpreters.create() + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + interp.exec_sync(script) + + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + @support.requires_fork() + def test_fork(self): + interp = interpreters.create() + import tempfile + with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + import os + try: + os.fork() + except RuntimeError: + with open('{file.name}', 'w', encoding='utf-8') as out: + out.write('{expected}') + """) + interp.exec_sync(script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + @unittest.skip('Fails on FreeBSD') + def test_already_running(self): + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.exec_sync('print("spam")') + + def test_bad_script(self): + interp = interpreters.create() + with self.assertRaises(TypeError): + interp.exec_sync(10) + + def test_bytes_for_script(self): + interp = interpreters.create() + with self.assertRaises(TypeError): + interp.exec_sync(b'print("spam")') + + def test_with_background_threads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + RAN = b'R' + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + os.write({w_interp}, {RAN!r}) + """) + interp.exec_sync(f"""if True: + os.write({w_interp}, {RAN!r}) + """) + + os.write(w_thread, DONE) + interp.exec_sync('t.join()') + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), FINISHED) + + # test_xxsubinterpreters covers the remaining + # Interpreter.exec_sync() behavior. + + +class TestInterpreterRun(TestBase): + + def test_success(self): + interp = interpreters.create() + script, file = _captured_script('print("it worked!", end="")') + with file: + t = interp.run(script) + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_failure(self): + caught = False + def excepthook(args): + nonlocal caught + caught = True + threading.excepthook = excepthook + try: + interp = interpreters.create() + t = interp.run('raise Exception') + t.join() + + self.assertTrue(caught) + except BaseException: + threading.excepthook = threading.__excepthook__ + + +class TestIsShareable(TestBase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + 'spam', + 10, + -10, + True, + False, + 100.0, + (), + (1, ('spam', 'eggs'), True), + ] + for obj in shareables: + with self.subTest(obj): + shareable = interpreters.is_shareable(obj) + self.assertTrue(shareable) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(repr(obj)): + self.assertFalse( + interpreters.is_shareable(obj)) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py new file mode 100644 index 00000000000000..3c3e18832d4168 --- /dev/null +++ b/Lib/test/test_interpreters/test_channels.py @@ -0,0 +1,328 @@ +import threading +from textwrap import dedent +import unittest +import time + +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +_channels = import_helper.import_module('_xxinterpchannels') +from test.support import interpreters +from test.support.interpreters import channels +from .utils import _run_output, TestBase + + +class TestChannels(TestBase): + + def test_create(self): + r, s = channels.create() + self.assertIsInstance(r, channels.RecvChannel) + self.assertIsInstance(s, channels.SendChannel) + + def test_list_all(self): + self.assertEqual(channels.list_all(), []) + created = set() + for _ in range(3): + ch = channels.create() + created.add(ch) + after = set(channels.list_all()) + self.assertEqual(after, created) + + def test_shareable(self): + rch, sch = channels.create() + + self.assertTrue( + interpreters.is_shareable(rch)) + self.assertTrue( + interpreters.is_shareable(sch)) + + sch.send_nowait(rch) + sch.send_nowait(sch) + rch2 = rch.recv() + sch2 = rch.recv() + + self.assertEqual(rch2, rch) + self.assertEqual(sch2, sch) + + def test_is_closed(self): + rch, sch = channels.create() + rbefore = rch.is_closed + sbefore = sch.is_closed + rch.close() + rafter = rch.is_closed + safter = sch.is_closed + + self.assertFalse(rbefore) + self.assertFalse(sbefore) + self.assertTrue(rafter) + self.assertTrue(safter) + + +class TestRecvChannelAttrs(TestBase): + + def test_id_type(self): + rch, _ = channels.create() + self.assertIsInstance(rch.id, _channels.ChannelID) + + def test_custom_id(self): + rch = channels.RecvChannel(1) + self.assertEqual(rch.id, 1) + + with self.assertRaises(TypeError): + channels.RecvChannel('1') + + def test_id_readonly(self): + rch = channels.RecvChannel(1) + with self.assertRaises(AttributeError): + rch.id = 2 + + def test_equality(self): + ch1, _ = channels.create() + ch2, _ = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + +class TestSendChannelAttrs(TestBase): + + def test_id_type(self): + _, sch = channels.create() + self.assertIsInstance(sch.id, _channels.ChannelID) + + def test_custom_id(self): + sch = channels.SendChannel(1) + self.assertEqual(sch.id, 1) + + with self.assertRaises(TypeError): + channels.SendChannel('1') + + def test_id_readonly(self): + sch = channels.SendChannel(1) + with self.assertRaises(AttributeError): + sch.id = 2 + + def test_equality(self): + _, ch1 = channels.create() + _, ch2 = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + +class TestSendRecv(TestBase): + + def test_send_recv_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + interp = interpreters.create() + interp.exec_sync(dedent(""" + from test.support.interpreters import channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_send_recv_different_threads(self): + r, s = channels.create() + + def f(): + while True: + try: + obj = r.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + s.send(obj) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + s.send(orig) + obj = r.recv() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main_with_default(self): + r, _ = channels.create() + obj = r.recv_nowait(None) + + self.assertIsNone(obj) + + def test_send_recv_nowait_same_interpreter(self): + interp = interpreters.create() + interp.exec_sync(dedent(""" + from test.support.interpreters import channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + assert obj == orig, 'expected: obj == orig' + # When going back to the same interpreter we get the same object. + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_nowait_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv_nowait() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv_nowait() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_recv_timeout(self): + r, _ = channels.create() + with self.assertRaises(TimeoutError): + r.recv(timeout=1) + + def test_recv_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv() + + def test_send_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send(b'spam') + + def test_recv_nowait_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv_nowait() + + def test_send_nowait_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send_nowait(b'spam') + + def test_recv_nowait_empty(self): + ch, _ = channels.create() + with self.assertRaises(channels.ChannelEmptyError): + ch.recv_nowait() + + def test_recv_nowait_default(self): + default = object() + rch, sch = channels.create() + obj1 = rch.recv_nowait(default) + sch.send_nowait(None) + sch.send_nowait(1) + sch.send_nowait(b'spam') + sch.send_nowait(b'eggs') + obj2 = rch.recv_nowait(default) + obj3 = rch.recv_nowait(default) + obj4 = rch.recv_nowait() + obj5 = rch.recv_nowait(default) + obj6 = rch.recv_nowait(default) + + self.assertIs(obj1, default) + self.assertIs(obj2, None) + self.assertEqual(obj3, 1) + self.assertEqual(obj4, b'spam') + self.assertEqual(obj5, b'eggs') + self.assertIs(obj6, default) + + def test_send_buffer(self): + buf = bytearray(b'spamspamspam') + obj = None + rch, sch = channels.create() + + def f(): + nonlocal obj + while True: + try: + obj = rch.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + t = threading.Thread(target=f) + t.start() + + sch.send_buffer(buf) + t.join() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + def test_send_buffer_nowait(self): + buf = bytearray(b'spamspamspam') + rch, sch = channels.create() + sch.send_buffer_nowait(buf) + obj = rch.recv() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py new file mode 100644 index 00000000000000..c2917d839904f9 --- /dev/null +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -0,0 +1,189 @@ +import contextlib +import json +import os +import os.path +import sys +from textwrap import dedent +import unittest + +from test import support +from test.support import import_helper +from test.support import os_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxsubinterpreters') +from .utils import TestBase + + +class StartupTests(TestBase): + + # We want to ensure the initial state of subinterpreters + # matches expectations. + + _subtest_count = 0 + + @contextlib.contextmanager + def subTest(self, *args): + with super().subTest(*args) as ctx: + self._subtest_count += 1 + try: + yield ctx + finally: + if self._debugged_in_subtest: + if self._subtest_count == 1: + # The first subtest adds a leading newline, so we + # compensate here by not printing a trailing newline. + print('### end subtest debug ###', end='') + else: + print('### end subtest debug ###') + self._debugged_in_subtest = False + + def debug(self, msg, *, header=None): + if header: + self._debug(f'--- {header} ---') + if msg: + if msg.endswith(os.linesep): + self._debug(msg[:-len(os.linesep)]) + else: + self._debug(msg) + self._debug('') + self._debug('------') + else: + self._debug(msg) + + _debugged = False + _debugged_in_subtest = False + def _debug(self, msg): + if not self._debugged: + print() + self._debugged = True + if self._subtest is not None: + if True: + if not self._debugged_in_subtest: + self._debugged_in_subtest = True + print('### start subtest debug ###') + print(msg) + else: + print(msg) + + def create_temp_dir(self): + import tempfile + tmp = tempfile.mkdtemp(prefix='test_interpreters_') + tmp = os.path.realpath(tmp) + self.addCleanup(os_helper.rmtree, tmp) + return tmp + + def write_script(self, *path, text): + filename = os.path.join(*path) + dirname = os.path.dirname(filename) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(dedent(text)) + return filename + + @support.requires_subprocess() + def run_python(self, argv, *, cwd=None): + # This method is inspired by + # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. + import shlex + import subprocess + if isinstance(argv, str): + argv = shlex.split(argv) + argv = [sys.executable, *argv] + try: + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + except Exception as exc: + self.debug(f'# cmd: {shlex.join(argv)}') + if isinstance(exc, FileNotFoundError) and not exc.filename: + if os.path.exists(argv[0]): + exists = 'exists' + else: + exists = 'does not exist' + self.debug(f'{argv[0]} {exists}') + raise # re-raise + assert proc.stderr == '' or proc.returncode != 0, proc.stderr + if proc.returncode != 0 and support.verbose: + self.debug(f'# python3 {shlex.join(argv[1:])} failed:') + self.debug(proc.stdout, header='stdout') + self.debug(proc.stderr, header='stderr') + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, '') + return proc.stdout + + def test_sys_path_0(self): + # The main interpreter's sys.path[0] should be used by subinterpreters. + script = ''' + import sys + from test.support import interpreters + + orig = sys.path[0] + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import json + import sys + print(json.dumps({{ + 'main': {orig!r}, + 'sub': sys.path[0], + }}, indent=4), flush=True) + """) + ''' + # / + # pkg/ + # __init__.py + # __main__.py + # script.py + # script.py + cwd = self.create_temp_dir() + self.write_script(cwd, 'pkg', '__init__.py', text='') + self.write_script(cwd, 'pkg', '__main__.py', text=script) + self.write_script(cwd, 'pkg', 'script.py', text=script) + self.write_script(cwd, 'script.py', text=script) + + cases = [ + ('script.py', cwd), + ('-m script', cwd), + ('-m pkg', cwd), + ('-m pkg.script', cwd), + ('-c "import script"', ''), + ] + for argv, expected in cases: + with self.subTest(f'python3 {argv}'): + out = self.run_python(argv, cwd=cwd) + data = json.loads(out) + sp0_main, sp0_sub = data['main'], data['sub'] + self.assertEqual(sp0_sub, sp0_main) + self.assertEqual(sp0_sub, expected) + # XXX Also check them all with the -P cmdline flag? + + +class FinalizationTests(TestBase): + + def test_gh_109793(self): + # Make sure finalization finishes and the correct error code + # is reported, even when subinterpreters get cleaned up at the end. + import subprocess + argv = [sys.executable, '-c', '''if True: + from test.support import interpreters + interp = interpreters.create() + raise Exception + '''] + proc = subprocess.run(argv, capture_output=True, text=True) + self.assertIn('Traceback', proc.stderr) + if proc.returncode == 0 and support.verbose: + print() + print("--- cmd unexpected succeeded ---") + print(f"stdout:\n{proc.stdout}") + print(f"stderr:\n{proc.stderr}") + print("------") + self.assertEqual(proc.returncode, 1) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py new file mode 100644 index 00000000000000..2a8ca99c1f6e3f --- /dev/null +++ b/Lib/test/test_interpreters/test_queues.py @@ -0,0 +1,299 @@ +import threading +from textwrap import dedent +import unittest +import time + +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +_queues = import_helper.import_module('_xxinterpqueues') +from test.support import interpreters +from test.support.interpreters import queues +from .utils import _run_output, TestBase + + +class TestBase(TestBase): + def tearDown(self): + for qid in _queues.list_all(): + try: + _queues.destroy(qid) + except Exception: + pass + + +class QueueTests(TestBase): + + def test_create(self): + with self.subTest('vanilla'): + queue = queues.create() + self.assertEqual(queue.maxsize, 0) + + with self.subTest('small maxsize'): + queue = queues.create(3) + self.assertEqual(queue.maxsize, 3) + + with self.subTest('big maxsize'): + queue = queues.create(100) + self.assertEqual(queue.maxsize, 100) + + with self.subTest('no maxsize'): + queue = queues.create(0) + self.assertEqual(queue.maxsize, 0) + + with self.subTest('negative maxsize'): + queue = queues.create(-10) + self.assertEqual(queue.maxsize, -10) + + with self.subTest('bad maxsize'): + with self.assertRaises(TypeError): + queues.create('1') + + def test_shareable(self): + queue1 = queues.create() + + interp = interpreters.create() + interp.exec_sync(dedent(f""" + from test.support.interpreters import queues + queue1 = queues.Queue({queue1.id}) + """)); + + with self.subTest('same interpreter'): + queue2 = queues.create() + queue1.put(queue2) + queue3 = queue1.get() + self.assertIs(queue3, queue2) + + with self.subTest('from current interpreter'): + queue4 = queues.create() + queue1.put(queue4) + out = _run_output(interp, dedent(""" + queue4 = queue1.get() + print(queue4.id) + """)) + qid = int(out) + self.assertEqual(qid, queue4.id) + + with self.subTest('from subinterpreter'): + out = _run_output(interp, dedent(""" + queue5 = queues.create() + queue1.put(queue5) + print(queue5.id) + """)) + qid = int(out) + queue5 = queue1.get() + self.assertEqual(queue5.id, qid) + + def test_id_type(self): + queue = queues.create() + self.assertIsInstance(queue.id, int) + + def test_custom_id(self): + with self.assertRaises(queues.QueueNotFoundError): + queues.Queue(1_000_000) + + def test_id_readonly(self): + queue = queues.create() + with self.assertRaises(AttributeError): + queue.id = 1_000_000 + + def test_maxsize_readonly(self): + queue = queues.create(10) + with self.assertRaises(AttributeError): + queue.maxsize = 1_000_000 + + def test_hashable(self): + queue = queues.create() + expected = hash(queue.id) + actual = hash(queue) + self.assertEqual(actual, expected) + + def test_equality(self): + queue1 = queues.create() + queue2 = queues.create() + self.assertEqual(queue1, queue1) + self.assertNotEqual(queue1, queue2) + + +class TestQueueOps(TestBase): + + def test_empty(self): + queue = queues.create() + before = queue.empty() + queue.put(None) + during = queue.empty() + queue.get() + after = queue.empty() + + self.assertIs(before, True) + self.assertIs(during, False) + self.assertIs(after, True) + + def test_full(self): + expected = [False, False, False, True, False, False, False] + actual = [] + queue = queues.create(3) + for _ in range(3): + actual.append(queue.full()) + queue.put(None) + actual.append(queue.full()) + for _ in range(3): + queue.get() + actual.append(queue.full()) + + self.assertEqual(actual, expected) + + def test_qsize(self): + expected = [0, 1, 2, 3, 2, 3, 2, 1, 0, 1, 0] + actual = [] + queue = queues.create() + for _ in range(3): + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + for _ in range(3): + queue.get() + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + + self.assertEqual(actual, expected) + + def test_put_get_main(self): + expected = list(range(20)) + queue = queues.create() + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] + + self.assertEqual(actual, expected) + + def test_put_timeout(self): + queue = queues.create(2) + queue.put(None) + queue.put(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1) + queue.get() + queue.put(None) + + def test_put_nowait(self): + queue = queues.create(2) + queue.put_nowait(None) + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + queue.get() + queue.put_nowait(None) + + def test_get_timeout(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get(timeout=0.1) + + def test_get_nowait(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + def test_put_get_same_interpreter(self): + interp = interpreters.create() + interp.exec_sync(dedent(""" + from test.support.interpreters import queues + queue = queues.create() + orig = b'spam' + queue.put(orig) + obj = queue.get() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + def test_put_get_different_interpreters(self): + interp = interpreters.create() + queue1 = queues.create() + queue2 = queues.create() + self.assertEqual(len(queues.list_all()), 2) + + obj1 = b'spam' + queue1.put(obj1) + + out = _run_output( + interp, + dedent(f""" + from test.support.interpreters import queues + queue1 = queues.Queue({queue1.id}) + queue2 = queues.Queue({queue2.id}) + assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' + obj = queue1.get() + assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0' + assert obj == b'spam', 'expected: obj == obj1' + # When going to another interpreter we get a copy. + assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' + obj2 = b'eggs' + print(id(obj2)) + assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' + queue2.put(obj2) + assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' + """)) + self.assertEqual(len(queues.list_all()), 2) + self.assertEqual(queue1.qsize(), 0) + self.assertEqual(queue2.qsize(), 1) + + obj2 = queue2.get() + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_put_cleared_with_subinterpreter(self): + interp = interpreters.create() + queue = queues.create() + + out = _run_output( + interp, + dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + obj1 = b'spam' + obj2 = b'eggs' + queue.put(obj1) + queue.put(obj2) + """)) + self.assertEqual(queue.qsize(), 2) + + obj1 = queue.get() + self.assertEqual(obj1, b'spam') + self.assertEqual(queue.qsize(), 1) + + del interp + self.assertEqual(queue.qsize(), 0) + + def test_put_get_different_threads(self): + queue1 = queues.create() + queue2 = queues.create() + + def f(): + while True: + try: + obj = queue1.get(timeout=0.1) + break + except queues.QueueEmpty: + continue + queue2.put(obj) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + queue1.put(orig) + obj = queue2.get() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py new file mode 100644 index 00000000000000..3cc570b3bf7128 --- /dev/null +++ b/Lib/test/test_interpreters/test_stress.py @@ -0,0 +1,38 @@ +import threading +import unittest + +from test import support +from test.support import import_helper +from test.support import threading_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxsubinterpreters') +from test.support import interpreters +from .utils import TestBase + + +class StressTests(TestBase): + + # In these tests we generally want a lot of interpreters, + # but not so many that any test takes too long. + + @support.requires_resource('cpu') + def test_create_many_sequential(self): + alive = [] + for _ in range(100): + interp = interpreters.create() + alive.append(interp) + + @support.requires_resource('cpu') + def test_create_many_threaded(self): + alive = [] + def task(): + interp = interpreters.create() + alive.append(interp) + threads = (threading.Thread(target=task) for _ in range(200)) + with threading_helper.start_threads(threads): + pass + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py new file mode 100644 index 00000000000000..3a37ed09dd8943 --- /dev/null +++ b/Lib/test/test_interpreters/utils.py @@ -0,0 +1,147 @@ +import contextlib +import os +import os.path +import subprocess +import sys +import tempfile +import threading +from textwrap import dedent +import unittest + +from test import support +from test.support import os_helper + +from test.support import interpreters + + +def _captured_script(script): + r, w = os.pipe() + indented = script.replace('\n', '\n ') + wrapped = dedent(f""" + import contextlib + with open({w}, 'w', encoding='utf-8') as spipe: + with contextlib.redirect_stdout(spipe): + {indented} + """) + return wrapped, open(r, encoding='utf-8') + + +def clean_up_interpreters(): + for interp in interpreters.list_all(): + if interp.id == 0: # main + continue + try: + interp.close() + except RuntimeError: + pass # already destroyed + + +def _run_output(interp, request, init=None): + script, rpipe = _captured_script(request) + with rpipe: + if init: + interp.prepare_main(init) + interp.exec_sync(script) + return rpipe.read() + + +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interp.exec_sync(dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() + + +class TestBase(unittest.TestCase): + + def pipe(self): + def ensure_closed(fd): + try: + os.close(fd) + except OSError: + pass + r, w = os.pipe() + self.addCleanup(lambda: ensure_closed(r)) + self.addCleanup(lambda: ensure_closed(w)) + return r, w + + def temp_dir(self): + tempdir = tempfile.mkdtemp() + tempdir = os.path.realpath(tempdir) + self.addCleanup(lambda: os_helper.rmtree(tempdir)) + return tempdir + + def make_script(self, filename, dirname=None, text=None): + if text: + text = dedent(text) + if dirname is None: + dirname = self.temp_dir() + filename = os.path.join(dirname, filename) + + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(text or '') + return filename + + def make_module(self, name, pathentry=None, text=None): + if text: + text = dedent(text) + if pathentry is None: + pathentry = self.temp_dir() + else: + os.makedirs(pathentry, exist_ok=True) + *subnames, basename = name.split('.') + + dirname = pathentry + for subname in subnames: + dirname = os.path.join(dirname, subname) + if os.path.isdir(dirname): + pass + elif os.path.exists(dirname): + raise Exception(dirname) + else: + os.mkdir(dirname) + initfile = os.path.join(dirname, '__init__.py') + if not os.path.exists(initfile): + with open(initfile, 'w'): + pass + filename = os.path.join(dirname, basename + '.py') + + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(text or '') + return filename + + @support.requires_subprocess() + def run_python(self, *argv): + proc = subprocess.run( + [sys.executable, *argv], + capture_output=True, + text=True, + ) + return proc.returncode, proc.stdout, proc.stderr + + def assert_python_ok(self, *argv): + exitcode, stdout, stderr = self.run_python(*argv) + self.assertNotEqual(exitcode, 1) + return stdout, stderr + + def assert_python_failure(self, *argv): + exitcode, stdout, stderr = self.run_python(*argv) + self.assertNotEqual(exitcode, 0) + return stdout, stderr + + def tearDown(self): + clean_up_interpreters() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 09cced9baef99b..1d78876f2a1c84 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1654,7 +1654,8 @@ def test_truncate_on_read_only(self): class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader - @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " + @skip_if_sanitizer(memory=True, address=True, thread=True, + reason="sanitizer defaults to crashing " "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedReaderTest.test_constructor(self) @@ -2021,7 +2022,8 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " + @skip_if_sanitizer(memory=True, address=True, thread=True, + reason="sanitizer defaults to crashing " "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedWriterTest.test_constructor(self) @@ -2520,7 +2522,8 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " + @skip_if_sanitizer(memory=True, address=True, thread=True, + reason="sanitizer defaults to crashing " "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedRandomTest.test_constructor(self) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 512745e45350d1..9af0730ea98004 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -187,7 +187,11 @@ def test_batched(self): [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)]) self.assertEqual(list(batched('ABCDEFG', 1)), [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)]) + self.assertEqual(list(batched('ABCDEF', 2, strict=True)), + [('A', 'B'), ('C', 'D'), ('E', 'F')]) + with self.assertRaises(ValueError): # Incomplete batch when strict + list(batched('ABCDEFG', 3, strict=True)) with self.assertRaises(TypeError): # Too few arguments list(batched('ABCDEFG')) with self.assertRaises(TypeError): @@ -1152,6 +1156,78 @@ def test_pairwise(self): with self.assertRaises(TypeError): pairwise(None) # non-iterable argument + def test_pairwise_reenter(self): + def check(reenter_at, expected): + class I: + count = 0 + def __iter__(self): + return self + def __next__(self): + self.count +=1 + if self.count in reenter_at: + return next(it) + return [self.count] # new object + + it = pairwise(I()) + for item in expected: + self.assertEqual(next(it), item) + + check({1}, [ + (([2], [3]), [4]), + ([4], [5]), + ]) + check({2}, [ + ([1], ([1], [3])), + (([1], [3]), [4]), + ([4], [5]), + ]) + check({3}, [ + ([1], [2]), + ([2], ([2], [4])), + (([2], [4]), [5]), + ([5], [6]), + ]) + check({1, 2}, [ + ((([3], [4]), [5]), [6]), + ([6], [7]), + ]) + check({1, 3}, [ + (([2], ([2], [4])), [5]), + ([5], [6]), + ]) + check({1, 4}, [ + (([2], [3]), (([2], [3]), [5])), + ((([2], [3]), [5]), [6]), + ([6], [7]), + ]) + check({2, 3}, [ + ([1], ([1], ([1], [4]))), + (([1], ([1], [4])), [5]), + ([5], [6]), + ]) + + def test_pairwise_reenter2(self): + def check(maxcount, expected): + class I: + count = 0 + def __iter__(self): + return self + def __next__(self): + if self.count >= maxcount: + raise StopIteration + self.count +=1 + if self.count == 1: + return next(it, None) + return [self.count] # new object + + it = pairwise(I()) + self.assertEqual(list(it), expected) + + check(1, []) + check(2, []) + check(3, []) + check(4, [(([2], [3]), [4])]) + def test_product(self): for args, result in [ ([], [()]), # zero iterables diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index efc982e8b0eb04..d6bce605e21463 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -143,11 +143,11 @@ def test_unexpected_data(self): ('{"spam":[}', 'Expecting value', 9), ('[42:', "Expecting ',' delimiter", 3), ('[42 "spam"', "Expecting ',' delimiter", 4), - ('[42,]', 'Expecting value', 4), + ('[42,]', "Illegal trailing comma before end of array", 3), ('{"spam":[42}', "Expecting ',' delimiter", 11), ('["]', 'Unterminated string starting at', 1), ('["spam":', "Expecting ',' delimiter", 7), - ('["spam",]', 'Expecting value', 8), + ('["spam",]', "Illegal trailing comma before end of array", 7), ('{:', 'Expecting property name enclosed in double quotes', 1), ('{,', 'Expecting property name enclosed in double quotes', 1), ('{42', 'Expecting property name enclosed in double quotes', 1), @@ -159,7 +159,9 @@ def test_unexpected_data(self): ('[{"spam":]', 'Expecting value', 9), ('{"spam":42 "ham"', "Expecting ',' delimiter", 11), ('[{"spam":42]', "Expecting ',' delimiter", 11), - ('{"spam":42,}', 'Expecting property name enclosed in double quotes', 11), + ('{"spam":42,}', "Illegal trailing comma before end of object", 10), + ('{"spam":42 , }', "Illegal trailing comma before end of object", 11), + ('[123 , ]', "Illegal trailing comma before end of array", 6), ] for data, msg, idx in test_cases: with self.assertRaises(self.JSONDecodeError) as cm: diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 9919d7fbe54ef7..164ff2013eb552 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -85,10 +85,10 @@ def test_highly_nested_objects_encoding(self): for x in range(100000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(l) with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(d) def test_endless_recursion(self): @@ -99,7 +99,7 @@ def default(self, o): return [o] with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(1000): EndlessJSONEncoder(check_circular=False).encode(5j) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 89be432e4b78c0..33db5d6443f6fd 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3922,6 +3922,25 @@ def test_90195(self): # Logger should be enabled, since explicitly mentioned self.assertFalse(logger.disabled) + def test_111615(self): + # See gh-111615 + import multiprocessing as mp + + config = { + 'version': 1, + 'handlers': { + 'sink': { + 'class': 'logging.handlers.QueueHandler', + 'queue': mp.get_context('spawn').Queue(), + }, + }, + 'root': { + 'handlers': ['sink'], + 'level': 'DEBUG', + }, + } + logging.config.dictConfig(config) + class ManagerTest(BaseTest): def test_manager_loggerclass(self): logged = [] diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 23fcbfac1f9c89..8c350eb02ccc17 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -681,6 +681,20 @@ def test_initialize_existing(self): self._box = mailbox.Maildir(self._path) self._check_basics() + def test_filename_leading_dot(self): + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + for subdir in 'tmp', 'new', 'cur': + fname = os.path.join(self._path, subdir, '.foo' + subdir) + with open(fname, 'wb') as f: + f.write(b"@") + self._box = mailbox.Maildir(self._path) + self.assertNotIn('.footmp', self._box) + self.assertNotIn('.foonew', self._box) + self.assertNotIn('.foocur', self._box) + self.assertEqual(list(self._box.iterkeys()), []) + def _check_basics(self, factory=None): # (Used by test_open_new() and test_open_existing().) self.assertEqual(self._box._path, os.path.abspath(self._path)) @@ -1333,6 +1347,19 @@ def test_sequences(self): self._box.remove(key1) self.assertEqual(self._box.get_sequences(), {'flagged':[key0]}) + self._box.set_sequences({'foo':[key0]}) + self.assertEqual(self._box.get_sequences(), {'foo':[key0]}) + + def test_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + self.assertEqual(box.get_sequences(), {}) + self.assertEqual(os.listdir(path), []) + box.set_sequences({}) + self.assertEqual(os.listdir(path), ['.mh_sequences']) + def test_issue2625(self): msg0 = mailbox.MHMessage(self._template % 0) msg0.add_sequence('foo') diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 731299294e6877..8192502a40791b 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -6,10 +6,12 @@ import unittest from test import support +import gc import io import _pyio as pyio import pickle import sys +import weakref class IntLike: def __init__(self, num): @@ -477,6 +479,25 @@ def test_getbuffer_empty(self): buf2.release() memio.write(b'x') + def test_getbuffer_gc_collect(self): + memio = self.ioclass(b"1234567890") + buf = memio.getbuffer() + memiowr = weakref.ref(memio) + bufwr = weakref.ref(buf) + # Create a reference loop. + a = [buf] + a.append(a) + # The Python implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + del memio + del buf + del a + # The C implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + gc.collect() + self.assertIsNone(memiowr()) + self.assertIsNone(bufwr()) + def test_read1(self): buf = self.buftype("1234567890") self.assertEqual(self.ioclass(buf).read1(), buf) diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index db2133a9e8d17b..98d1cbe824df12 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -1,4 +1,5 @@ # Test the module type +import importlib.machinery import unittest import weakref from test.support import gc_collect @@ -29,7 +30,7 @@ def test_uninitialized(self): self.fail("__name__ = %s" % repr(s)) except AttributeError: pass - self.assertEqual(foo.__doc__, ModuleType.__doc__) + self.assertEqual(foo.__doc__, ModuleType.__doc__ or '') def test_uninitialized_missing_getattr(self): # Issue 8297 @@ -264,6 +265,35 @@ def test_module_repr_source(self): self.assertEqual(r[-len(ends_with):], ends_with, '{!r} does not end with {!r}'.format(r, ends_with)) + def test_module_repr_with_namespace_package(self): + m = ModuleType('foo') + loader = importlib.machinery.NamespaceLoader('foo', ['bar'], 'baz') + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + self.assertEqual(repr(m), "") + + def test_module_repr_with_namespace_package_and_custom_loader(self): + m = ModuleType('foo') + loader = BareLoader() + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + expected_repr_pattern = r"\)>" + self.assertRegex(repr(m), expected_repr_pattern) + self.assertNotIn('from', repr(m)) + + def test_module_repr_with_fake_namespace_package(self): + m = ModuleType('foo') + loader = BareLoader() + loader._path = ['spam'] + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + expected_repr_pattern = r"\)>" + self.assertRegex(repr(m), expected_repr_pattern) + self.assertNotIn('from', repr(m)) + def test_module_finalization_at_shutdown(self): # Module globals and builtins should still be available during shutdown rc, out, err = assert_python_ok("-c", "from test.test_module import final_a") diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 3e710d1c6dabe4..bf990ed36fbcae 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -256,6 +256,7 @@ def test_join(self): tester('ntpath.join("a", "b", "c")', 'a\\b\\c') tester('ntpath.join("a\\", "b", "c")', 'a\\b\\c') tester('ntpath.join("a", "b\\", "c")', 'a\\b\\c') + tester('ntpath.join("a", "b", "c\\")', 'a\\b\\c\\') tester('ntpath.join("a", "b", "\\c")', '\\c') tester('ntpath.join("d:\\", "\\pleep")', 'd:\\pleep') tester('ntpath.join("d:\\", "a", "b")', 'd:\\a\\b') @@ -313,6 +314,16 @@ def test_join(self): tester("ntpath.join('\\\\computer\\', 'share')", '\\\\computer\\share') tester("ntpath.join('\\\\computer\\share\\', 'a')", '\\\\computer\\share\\a') tester("ntpath.join('\\\\computer\\share\\a\\', 'b')", '\\\\computer\\share\\a\\b') + # Second part is anchored, so that the first part is ignored. + tester("ntpath.join('a', 'Z:b', 'c')", 'Z:b\\c') + tester("ntpath.join('a', 'Z:\\b', 'c')", 'Z:\\b\\c') + tester("ntpath.join('a', '\\\\b\\c', 'd')", '\\\\b\\c\\d') + # Second part has a root but not drive. + tester("ntpath.join('a', '\\b', 'c')", '\\b\\c') + tester("ntpath.join('Z:/a', '/b', 'c')", 'Z:\\b\\c') + tester("ntpath.join('//?/Z:/a', '/b', 'c')", '\\\\?\\Z:\\b\\c') + tester("ntpath.join('D:a', './c:b')", 'D:a\\.\\c:b') + tester("ntpath.join('D:/a', './c:b')", 'D:\\a\\.\\c:b') def test_normpath(self): tester("ntpath.normpath('A//////././//.//B')", r'A\B') diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index c31c9684051196..c66c5797471413 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1743,7 +1743,7 @@ def tearDown(self): os.removedirs(path) -@os_helper.skip_unless_working_chmod +@unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") class ChownFileTests(unittest.TestCase): @classmethod @@ -5080,7 +5080,10 @@ def test_fork(self): support.wait_process(pid, exitcode=0) """ assert_python_ok("-c", code) - assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") + if support.Py_GIL_DISABLED: + assert_python_ok("-c", code, PYTHONMALLOC="mimalloc_debug") + else: + assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") @unittest.skipUnless(sys.platform in ("linux", "darwin"), "Only Linux and macOS detect this today.") diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py deleted file mode 100644 index 427e082f3e16cb..00000000000000 --- a/Lib/test/test_pathlib.py +++ /dev/null @@ -1,3797 +0,0 @@ -import collections.abc -import io -import os -import sys -import errno -import pathlib -import pickle -import posixpath -import socket -import stat -import tempfile -import unittest -from unittest import mock -from urllib.request import pathname2url - -from test.support import import_helper -from test.support import set_recursion_limit -from test.support import is_emscripten, is_wasi -from test.support import os_helper -from test.support.os_helper import TESTFN, FakePath - -try: - import grp, pwd -except ImportError: - grp = pwd = None - - -class UnsupportedOperationTest(unittest.TestCase): - def test_is_notimplemented(self): - self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) - - -# Make sure any symbolic links in the base test path are resolved. -BASE = os.path.realpath(TESTFN) -join = lambda *x: os.path.join(BASE, *x) -rel_join = lambda *x: os.path.join(TESTFN, *x) - -only_nt = unittest.skipIf(os.name != 'nt', - 'test requires a Windows-compatible system') -only_posix = unittest.skipIf(os.name == 'nt', - 'test requires a POSIX-compatible system') - - -# -# Tests for the pure classes. -# - -class PurePathTest(unittest.TestCase): - cls = pathlib.PurePath - - # Keys are canonical paths, values are list of tuples of arguments - # supposed to produce equal paths. - equivalences = { - 'a/b': [ - ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), - ('a/b/',), ('a//b',), ('a//b//',), - # Empty components get removed. - ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), - ], - '/b/c/d': [ - ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), - # Empty components get removed. - ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), - ], - } - - def setUp(self): - p = self.cls('a') - self.pathmod = p.pathmod - self.sep = self.pathmod.sep - self.altsep = self.pathmod.altsep - - def test_constructor_common(self): - P = self.cls - p = P('a') - self.assertIsInstance(p, P) - P('a', 'b', 'c') - P('/a', 'b', 'c') - P('a/b/c') - P('/a/b/c') - P(FakePath("a/b/c")) - self.assertEqual(P(P('a')), P('a')) - self.assertEqual(P(P('a'), 'b'), P('a/b')) - self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) - self.assertEqual(P(P('./a:b')), P('./a:b')) - - def test_concrete_class(self): - if self.cls is pathlib.PurePath: - expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath - else: - expected = self.cls - p = self.cls('a') - self.assertIs(type(p), expected) - - def test_different_pathmods_unequal(self): - p = self.cls('a') - if p.pathmod is posixpath: - q = pathlib.PureWindowsPath('a') - else: - q = pathlib.PurePosixPath('a') - self.assertNotEqual(p, q) - - def test_different_pathmods_unordered(self): - p = self.cls('a') - if p.pathmod is posixpath: - q = pathlib.PureWindowsPath('a') - else: - q = pathlib.PurePosixPath('a') - with self.assertRaises(TypeError): - p < q - with self.assertRaises(TypeError): - p <= q - with self.assertRaises(TypeError): - p > q - with self.assertRaises(TypeError): - p >= q - - def _check_str_subclass(self, *args): - # Issue #21127: it should be possible to construct a PurePath object - # from a str subclass instance, and it then gets converted to - # a pure str object. - class StrSubclass(str): - pass - P = self.cls - p = P(*(StrSubclass(x) for x in args)) - self.assertEqual(p, P(*args)) - for part in p.parts: - self.assertIs(type(part), str) - - def test_str_subclass_common(self): - self._check_str_subclass('') - self._check_str_subclass('.') - self._check_str_subclass('a') - self._check_str_subclass('a/b.txt') - self._check_str_subclass('/a/b.txt') - - def test_with_segments_common(self): - class P(self.cls): - def __init__(self, *pathsegments, session_id): - super().__init__(*pathsegments) - self.session_id = session_id - - def with_segments(self, *pathsegments): - return type(self)(*pathsegments, session_id=self.session_id) - p = P('foo', 'bar', session_id=42) - self.assertEqual(42, (p / 'foo').session_id) - self.assertEqual(42, ('foo' / p).session_id) - self.assertEqual(42, p.joinpath('foo').session_id) - self.assertEqual(42, p.with_name('foo').session_id) - self.assertEqual(42, p.with_stem('foo').session_id) - self.assertEqual(42, p.with_suffix('.foo').session_id) - self.assertEqual(42, p.with_segments('foo').session_id) - self.assertEqual(42, p.relative_to('foo').session_id) - self.assertEqual(42, p.parent.session_id) - for parent in p.parents: - self.assertEqual(42, parent.session_id) - - def _get_drive_root_parts(self, parts): - path = self.cls(*parts) - return path.drive, path.root, path.parts - - def _check_drive_root_parts(self, arg, *expected): - sep = self.pathmod.sep - actual = self._get_drive_root_parts([x.replace('/', sep) for x in arg]) - self.assertEqual(actual, expected) - if altsep := self.pathmod.altsep: - actual = self._get_drive_root_parts([x.replace('/', altsep) for x in arg]) - self.assertEqual(actual, expected) - - def test_drive_root_parts_common(self): - check = self._check_drive_root_parts - sep = self.pathmod.sep - # Unanchored parts. - check((), '', '', ()) - check(('a',), '', '', ('a',)) - check(('a/',), '', '', ('a',)) - check(('a', 'b'), '', '', ('a', 'b')) - # Expansion. - check(('a/b',), '', '', ('a', 'b')) - check(('a/b/',), '', '', ('a', 'b')) - check(('a', 'b/c', 'd'), '', '', ('a', 'b', 'c', 'd')) - # Collapsing and stripping excess slashes. - check(('a', 'b//c', 'd'), '', '', ('a', 'b', 'c', 'd')) - check(('a', 'b/c/', 'd'), '', '', ('a', 'b', 'c', 'd')) - # Eliminating standalone dots. - check(('.',), '', '', ()) - check(('.', '.', 'b'), '', '', ('b',)) - check(('a', '.', 'b'), '', '', ('a', 'b')) - check(('a', '.', '.'), '', '', ('a',)) - # The first part is anchored. - check(('/a/b',), '', sep, (sep, 'a', 'b')) - check(('/a', 'b'), '', sep, (sep, 'a', 'b')) - check(('/a/', 'b'), '', sep, (sep, 'a', 'b')) - # Ignoring parts before an anchored part. - check(('a', '/b', 'c'), '', sep, (sep, 'b', 'c')) - check(('a', '/b', '/c'), '', sep, (sep, 'c')) - - def test_join_common(self): - P = self.cls - p = P('a/b') - pp = p.joinpath('c') - self.assertEqual(pp, P('a/b/c')) - self.assertIs(type(pp), type(p)) - pp = p.joinpath('c', 'd') - self.assertEqual(pp, P('a/b/c/d')) - pp = p.joinpath(P('c')) - self.assertEqual(pp, P('a/b/c')) - pp = p.joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div_common(self): - # Basically the same as joinpath(). - P = self.cls - p = P('a/b') - pp = p / 'c' - self.assertEqual(pp, P('a/b/c')) - self.assertIs(type(pp), type(p)) - pp = p / 'c/d' - self.assertEqual(pp, P('a/b/c/d')) - pp = p / 'c' / 'd' - self.assertEqual(pp, P('a/b/c/d')) - pp = 'c' / p / 'd' - self.assertEqual(pp, P('c/a/b/d')) - pp = p / P('c') - self.assertEqual(pp, P('a/b/c')) - pp = p/ '/c' - self.assertEqual(pp, P('/c')) - - def _check_str(self, expected, args): - p = self.cls(*args) - self.assertEqual(str(p), expected.replace('/', self.sep)) - - def test_str_common(self): - # Canonicalized paths roundtrip. - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self._check_str(pathstr, (pathstr,)) - # Special case for the empty path. - self._check_str('.', ('',)) - # Other tests for str() are in test_equivalences(). - - def test_as_posix_common(self): - P = self.cls - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self.assertEqual(P(pathstr).as_posix(), pathstr) - # Other tests for as_posix() are in test_equivalences(). - - def test_repr_common(self): - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - with self.subTest(pathstr=pathstr): - p = self.cls(pathstr) - clsname = p.__class__.__name__ - r = repr(p) - # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) - inner = r[len(clsname) + 1 : -1] - self.assertEqual(eval(inner), p.as_posix()) - - def test_eq_common(self): - P = self.cls - self.assertEqual(P('a/b'), P('a/b')) - self.assertEqual(P('a/b'), P('a', 'b')) - self.assertNotEqual(P('a/b'), P('a')) - self.assertNotEqual(P('a/b'), P('/a/b')) - self.assertNotEqual(P('a/b'), P()) - self.assertNotEqual(P('/a/b'), P('/')) - self.assertNotEqual(P(), P('/')) - self.assertNotEqual(P(), "") - self.assertNotEqual(P(), {}) - self.assertNotEqual(P(), int) - - def test_match_common(self): - P = self.cls - self.assertRaises(ValueError, P('a').match, '') - self.assertRaises(ValueError, P('a').match, '.') - # Simple relative pattern. - self.assertTrue(P('b.py').match('b.py')) - self.assertTrue(P('a/b.py').match('b.py')) - self.assertTrue(P('/a/b.py').match('b.py')) - self.assertFalse(P('a.py').match('b.py')) - self.assertFalse(P('b/py').match('b.py')) - self.assertFalse(P('/a.py').match('b.py')) - self.assertFalse(P('b.py/c').match('b.py')) - # Wildcard relative pattern. - self.assertTrue(P('b.py').match('*.py')) - self.assertTrue(P('a/b.py').match('*.py')) - self.assertTrue(P('/a/b.py').match('*.py')) - self.assertFalse(P('b.pyc').match('*.py')) - self.assertFalse(P('b./py').match('*.py')) - self.assertFalse(P('b.py/c').match('*.py')) - # Multi-part relative pattern. - self.assertTrue(P('ab/c.py').match('a*/*.py')) - self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) - self.assertFalse(P('a.py').match('a*/*.py')) - self.assertFalse(P('/dab/c.py').match('a*/*.py')) - self.assertFalse(P('ab/c.py/d').match('a*/*.py')) - # Absolute pattern. - self.assertTrue(P('/b.py').match('/*.py')) - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('a/b.py').match('/*.py')) - self.assertFalse(P('/a/b.py').match('/*.py')) - # Multi-part absolute pattern. - self.assertTrue(P('/a/b.py').match('/a/*.py')) - self.assertFalse(P('/ab.py').match('/a/*.py')) - self.assertFalse(P('/a/b/c.py').match('/a/*.py')) - # Multi-part glob-style pattern. - self.assertTrue(P('a').match('**')) - self.assertTrue(P('c.py').match('**')) - self.assertTrue(P('a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('**')) - self.assertTrue(P('/a/b/c.py').match('/**')) - self.assertTrue(P('/a/b/c.py').match('**/')) - self.assertTrue(P('/a/b/c.py').match('/a/**')) - self.assertTrue(P('/a/b/c.py').match('**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/b/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/**/**/**/**/*.py')) - self.assertFalse(P('c.py').match('**/a.py')) - self.assertFalse(P('c.py').match('c/**')) - self.assertFalse(P('a/b/c.py').match('**/a')) - self.assertFalse(P('a/b/c.py').match('**/a/b')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c.')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) - self.assertFalse(P('a/b/c.py').match('/a/b/c.py/**')) - self.assertFalse(P('a/b/c.py').match('/**/a/b/c.py')) - self.assertRaises(ValueError, P('a').match, '**a/b/c') - self.assertRaises(ValueError, P('a').match, 'a/b/c**') - # Case-sensitive flag - self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) - self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) - self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) - self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) - # Matching against empty path - self.assertFalse(P().match('*')) - self.assertTrue(P().match('**')) - self.assertFalse(P().match('**/*')) - - def test_parts_common(self): - # `parts` returns a tuple. - sep = self.sep - P = self.cls - p = P('a/b') - parts = p.parts - self.assertEqual(parts, ('a', 'b')) - # When the path is absolute, the anchor is a separate part. - p = P('/a/b') - parts = p.parts - self.assertEqual(parts, (sep, 'a', 'b')) - - def test_equivalences(self): - for k, tuples in self.equivalences.items(): - canon = k.replace('/', self.sep) - posix = k.replace(self.sep, '/') - if canon != posix: - tuples = tuples + [ - tuple(part.replace('/', self.sep) for part in t) - for t in tuples - ] - tuples.append((posix, )) - pcanon = self.cls(canon) - for t in tuples: - p = self.cls(*t) - self.assertEqual(p, pcanon, "failed with args {}".format(t)) - self.assertEqual(hash(p), hash(pcanon)) - self.assertEqual(str(p), canon) - self.assertEqual(p.as_posix(), posix) - - def test_parent_common(self): - # Relative - P = self.cls - p = P('a/b/c') - self.assertEqual(p.parent, P('a/b')) - self.assertEqual(p.parent.parent, P('a')) - self.assertEqual(p.parent.parent.parent, P()) - self.assertEqual(p.parent.parent.parent.parent, P()) - # Anchored - p = P('/a/b/c') - self.assertEqual(p.parent, P('/a/b')) - self.assertEqual(p.parent.parent, P('/a')) - self.assertEqual(p.parent.parent.parent, P('/')) - self.assertEqual(p.parent.parent.parent.parent, P('/')) - - def test_parents_common(self): - # Relative - P = self.cls - p = P('a/b/c') - par = p.parents - self.assertEqual(len(par), 3) - self.assertEqual(par[0], P('a/b')) - self.assertEqual(par[1], P('a')) - self.assertEqual(par[2], P('.')) - self.assertEqual(par[-1], P('.')) - self.assertEqual(par[-2], P('a')) - self.assertEqual(par[-3], P('a/b')) - self.assertEqual(par[0:1], (P('a/b'),)) - self.assertEqual(par[:2], (P('a/b'), P('a'))) - self.assertEqual(par[:-1], (P('a/b'), P('a'))) - self.assertEqual(par[1:], (P('a'), P('.'))) - self.assertEqual(par[::2], (P('a/b'), P('.'))) - self.assertEqual(par[::-1], (P('.'), P('a'), P('a/b'))) - self.assertEqual(list(par), [P('a/b'), P('a'), P('.')]) - with self.assertRaises(IndexError): - par[-4] - with self.assertRaises(IndexError): - par[3] - with self.assertRaises(TypeError): - par[0] = p - # Anchored - p = P('/a/b/c') - par = p.parents - self.assertEqual(len(par), 3) - self.assertEqual(par[0], P('/a/b')) - self.assertEqual(par[1], P('/a')) - self.assertEqual(par[2], P('/')) - self.assertEqual(par[-1], P('/')) - self.assertEqual(par[-2], P('/a')) - self.assertEqual(par[-3], P('/a/b')) - self.assertEqual(par[0:1], (P('/a/b'),)) - self.assertEqual(par[:2], (P('/a/b'), P('/a'))) - self.assertEqual(par[:-1], (P('/a/b'), P('/a'))) - self.assertEqual(par[1:], (P('/a'), P('/'))) - self.assertEqual(par[::2], (P('/a/b'), P('/'))) - self.assertEqual(par[::-1], (P('/'), P('/a'), P('/a/b'))) - self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')]) - with self.assertRaises(IndexError): - par[-4] - with self.assertRaises(IndexError): - par[3] - - def test_drive_common(self): - P = self.cls - self.assertEqual(P('a/b').drive, '') - self.assertEqual(P('/a/b').drive, '') - self.assertEqual(P('').drive, '') - - def test_root_common(self): - P = self.cls - sep = self.sep - self.assertEqual(P('').root, '') - self.assertEqual(P('a/b').root, '') - self.assertEqual(P('/').root, sep) - self.assertEqual(P('/a/b').root, sep) - - def test_anchor_common(self): - P = self.cls - sep = self.sep - self.assertEqual(P('').anchor, '') - self.assertEqual(P('a/b').anchor, '') - self.assertEqual(P('/').anchor, sep) - self.assertEqual(P('/a/b').anchor, sep) - - def test_name_common(self): - P = self.cls - self.assertEqual(P('').name, '') - self.assertEqual(P('.').name, '') - self.assertEqual(P('/').name, '') - self.assertEqual(P('a/b').name, 'b') - self.assertEqual(P('/a/b').name, 'b') - self.assertEqual(P('/a/b/.').name, 'b') - self.assertEqual(P('a/b.py').name, 'b.py') - self.assertEqual(P('/a/b.py').name, 'b.py') - - def test_suffix_common(self): - P = self.cls - self.assertEqual(P('').suffix, '') - self.assertEqual(P('.').suffix, '') - self.assertEqual(P('..').suffix, '') - self.assertEqual(P('/').suffix, '') - self.assertEqual(P('a/b').suffix, '') - self.assertEqual(P('/a/b').suffix, '') - self.assertEqual(P('/a/b/.').suffix, '') - self.assertEqual(P('a/b.py').suffix, '.py') - self.assertEqual(P('/a/b.py').suffix, '.py') - self.assertEqual(P('a/.hgrc').suffix, '') - self.assertEqual(P('/a/.hgrc').suffix, '') - self.assertEqual(P('a/.hg.rc').suffix, '.rc') - self.assertEqual(P('/a/.hg.rc').suffix, '.rc') - self.assertEqual(P('a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') - - def test_suffixes_common(self): - P = self.cls - self.assertEqual(P('').suffixes, []) - self.assertEqual(P('.').suffixes, []) - self.assertEqual(P('/').suffixes, []) - self.assertEqual(P('a/b').suffixes, []) - self.assertEqual(P('/a/b').suffixes, []) - self.assertEqual(P('/a/b/.').suffixes, []) - self.assertEqual(P('a/b.py').suffixes, ['.py']) - self.assertEqual(P('/a/b.py').suffixes, ['.py']) - self.assertEqual(P('a/.hgrc').suffixes, []) - self.assertEqual(P('/a/.hgrc').suffixes, []) - self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem_common(self): - P = self.cls - self.assertEqual(P('').stem, '') - self.assertEqual(P('.').stem, '') - self.assertEqual(P('..').stem, '..') - self.assertEqual(P('/').stem, '') - self.assertEqual(P('a/b').stem, 'b') - self.assertEqual(P('a/b.py').stem, 'b') - self.assertEqual(P('a/.hgrc').stem, '.hgrc') - self.assertEqual(P('a/.hg.rc').stem, '.hg') - self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name_common(self): - P = self.cls - self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) - self.assertRaises(ValueError, P('').with_name, 'd.xml') - self.assertRaises(ValueError, P('.').with_name, 'd.xml') - self.assertRaises(ValueError, P('/').with_name, 'd.xml') - self.assertRaises(ValueError, P('a/b').with_name, '') - self.assertRaises(ValueError, P('a/b').with_name, '.') - self.assertRaises(ValueError, P('a/b').with_name, '/c') - self.assertRaises(ValueError, P('a/b').with_name, 'c/') - self.assertRaises(ValueError, P('a/b').with_name, 'c/d') - - def test_with_stem_common(self): - P = self.cls - self.assertEqual(P('a/b').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) - self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) - self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) - self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) - self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) - self.assertRaises(ValueError, P('').with_stem, 'd') - self.assertRaises(ValueError, P('.').with_stem, 'd') - self.assertRaises(ValueError, P('/').with_stem, 'd') - self.assertRaises(ValueError, P('a/b').with_stem, '') - self.assertRaises(ValueError, P('a/b').with_stem, '.') - self.assertRaises(ValueError, P('a/b').with_stem, '/c') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') - - def test_with_suffix_common(self): - P = self.cls - self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) - self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) - # Stripping suffix. - self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) - self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('a/b').with_suffix, '/') - self.assertRaises(ValueError, P('a/b').with_suffix, '.') - self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') - self.assertRaises(ValueError, P('a/b').with_suffix, './.d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') - - def test_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.relative_to) - self.assertRaises(TypeError, p.relative_to, b'a') - self.assertEqual(p.relative_to(P()), P('a/b')) - self.assertEqual(p.relative_to(''), P('a/b')) - self.assertEqual(p.relative_to(P('a')), P('b')) - self.assertEqual(p.relative_to('a'), P('b')) - self.assertEqual(p.relative_to('a/'), P('b')) - self.assertEqual(p.relative_to(P('a/b')), P()) - self.assertEqual(p.relative_to('a/b'), P()) - self.assertEqual(p.relative_to(P(), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('a/b', walk_up=True), P()) - self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) - # With several args. - with self.assertWarns(DeprecationWarning): - p.relative_to('a', 'b') - p.relative_to('a', 'b', walk_up=True) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('c')) - self.assertRaises(ValueError, p.relative_to, P('a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('a/c')) - self.assertRaises(ValueError, p.relative_to, P('/a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - p = P('/a/b') - self.assertEqual(p.relative_to(P('/')), P('a/b')) - self.assertEqual(p.relative_to('/'), P('a/b')) - self.assertEqual(p.relative_to(P('/a')), P('b')) - self.assertEqual(p.relative_to('/a'), P('b')) - self.assertEqual(p.relative_to('/a/'), P('b')) - self.assertEqual(p.relative_to(P('/a/b')), P()) - self.assertEqual(p.relative_to('/a/b'), P()) - self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P()) - self.assertEqual(p.relative_to('/a/b', walk_up=True), P()) - self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/c')) - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - - def test_is_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.is_relative_to) - self.assertRaises(TypeError, p.is_relative_to, b'a') - self.assertTrue(p.is_relative_to(P())) - self.assertTrue(p.is_relative_to('')) - self.assertTrue(p.is_relative_to(P('a'))) - self.assertTrue(p.is_relative_to('a/')) - self.assertTrue(p.is_relative_to(P('a/b'))) - self.assertTrue(p.is_relative_to('a/b')) - # With several args. - with self.assertWarns(DeprecationWarning): - p.is_relative_to('a', 'b') - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('c'))) - self.assertFalse(p.is_relative_to(P('a/b/c'))) - self.assertFalse(p.is_relative_to(P('a/c'))) - self.assertFalse(p.is_relative_to(P('/a'))) - p = P('/a/b') - self.assertTrue(p.is_relative_to(P('/'))) - self.assertTrue(p.is_relative_to('/')) - self.assertTrue(p.is_relative_to(P('/a'))) - self.assertTrue(p.is_relative_to('/a')) - self.assertTrue(p.is_relative_to('/a/')) - self.assertTrue(p.is_relative_to(P('/a/b'))) - self.assertTrue(p.is_relative_to('/a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/c'))) - self.assertFalse(p.is_relative_to(P('/a/b/c'))) - self.assertFalse(p.is_relative_to(P('/a/c'))) - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('a'))) - - def test_pickling_common(self): - P = self.cls - p = P('/a/b') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertIs(pp.__class__, p.__class__) - self.assertEqual(pp, p) - self.assertEqual(hash(pp), hash(p)) - self.assertEqual(str(pp), str(p)) - - def test_fspath_common(self): - P = self.cls - p = P('a/b') - self._check_str(p.__fspath__(), ('a/b',)) - self._check_str(os.fspath(p), ('a/b',)) - - def test_bytes(self): - P = self.cls - message = (r"argument should be a str or an os\.PathLike object " - r"where __fspath__ returns a str, not 'bytes'") - with self.assertRaisesRegex(TypeError, message): - P(b'a') - with self.assertRaisesRegex(TypeError, message): - P(b'a', 'b') - with self.assertRaisesRegex(TypeError, message): - P('a', b'b') - with self.assertRaises(TypeError): - P('a').joinpath(b'b') - with self.assertRaises(TypeError): - P('a') / b'b' - with self.assertRaises(TypeError): - b'a' / P('b') - with self.assertRaises(TypeError): - P('a').match(b'b') - with self.assertRaises(TypeError): - P('a').relative_to(b'b') - with self.assertRaises(TypeError): - P('a').with_name(b'b') - with self.assertRaises(TypeError): - P('a').with_stem(b'b') - with self.assertRaises(TypeError): - P('a').with_suffix(b'b') - - def test_as_bytes_common(self): - sep = os.fsencode(self.sep) - P = self.cls - self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') - - def test_ordering_common(self): - # Ordering is tuple-alike. - def assertLess(a, b): - self.assertLess(a, b) - self.assertGreater(b, a) - P = self.cls - a = P('a') - b = P('a/b') - c = P('abc') - d = P('b') - assertLess(a, b) - assertLess(a, c) - assertLess(a, d) - assertLess(b, c) - assertLess(c, d) - P = self.cls - a = P('/a') - b = P('/a/b') - c = P('/abc') - d = P('/b') - assertLess(a, b) - assertLess(a, c) - assertLess(a, d) - assertLess(b, c) - assertLess(c, d) - with self.assertRaises(TypeError): - P() < {} - - def test_as_uri_common(self): - P = self.cls - with self.assertRaises(ValueError): - P('a').as_uri() - with self.assertRaises(ValueError): - P().as_uri() - - def test_repr_roundtrips(self): - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - with self.subTest(pathstr=pathstr): - p = self.cls(pathstr) - r = repr(p) - # The repr() roundtrips. - q = eval(r, pathlib.__dict__) - self.assertIs(q.__class__, p.__class__) - self.assertEqual(q, p) - self.assertEqual(repr(q), r) - - -class PurePosixPathTest(PurePathTest): - cls = pathlib.PurePosixPath - - def test_drive_root_parts(self): - check = self._check_drive_root_parts - # Collapsing of excess leading slashes, except for the double-slash - # special case. - check(('//a', 'b'), '', '//', ('//', 'a', 'b')) - check(('///a', 'b'), '', '/', ('/', 'a', 'b')) - check(('////a', 'b'), '', '/', ('/', 'a', 'b')) - # Paths which look like NT paths aren't treated specially. - check(('c:a',), '', '', ('c:a',)) - check(('c:\\a',), '', '', ('c:\\a',)) - check(('\\a',), '', '', ('\\a',)) - - def test_root(self): - P = self.cls - self.assertEqual(P('/a/b').root, '/') - self.assertEqual(P('///a/b').root, '/') - # POSIX special case for two leading slashes. - self.assertEqual(P('//a/b').root, '//') - - def test_eq(self): - P = self.cls - self.assertNotEqual(P('a/b'), P('A/b')) - self.assertEqual(P('/a'), P('///a')) - self.assertNotEqual(P('/a'), P('//a')) - - def test_as_uri(self): - P = self.cls - self.assertEqual(P('/').as_uri(), 'file:///') - self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c') - self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c') - - def test_as_uri_non_ascii(self): - from urllib.parse import quote_from_bytes - P = self.cls - try: - os.fsencode('\xe9') - except UnicodeEncodeError: - self.skipTest("\\xe9 cannot be encoded to the filesystem encoding") - self.assertEqual(P('/a/b\xe9').as_uri(), - 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9'))) - - def test_match(self): - P = self.cls - self.assertFalse(P('A.py').match('a.PY')) - - def test_is_absolute(self): - P = self.cls - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertTrue(P('/').is_absolute()) - self.assertTrue(P('/a').is_absolute()) - self.assertTrue(P('/a/b/').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - self.assertIs(False, P('/dev/con/PRN/NUL').is_reserved()) - - def test_join(self): - P = self.cls - p = P('//a') - pp = p.joinpath('b') - self.assertEqual(pp, P('//a/b')) - pp = P('/a').joinpath('//c') - self.assertEqual(pp, P('//c')) - pp = P('//a').joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('//a') - pp = p / 'b' - self.assertEqual(pp, P('//a/b')) - pp = P('/a') / '//c' - self.assertEqual(pp, P('//c')) - pp = P('//a') / '/c' - self.assertEqual(pp, P('/c')) - - def test_parse_windows_path(self): - P = self.cls - p = P('c:', 'a', 'b') - pp = P(pathlib.PureWindowsPath('c:\\a\\b')) - self.assertEqual(p, pp) - - -class PureWindowsPathTest(PurePathTest): - cls = pathlib.PureWindowsPath - - equivalences = PurePathTest.equivalences.copy() - equivalences.update({ - './a:b': [ ('./a:b',) ], - 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], - 'c:/a': [ - ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), - ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), - ], - '//a/b/': [ ('//a/b',) ], - '//a/b/c': [ - ('//a/b', 'c'), ('//a/b/', 'c'), - ], - }) - - def test_drive_root_parts(self): - check = self._check_drive_root_parts - # First part is anchored. - check(('c:',), 'c:', '', ('c:',)) - check(('c:/',), 'c:', '\\', ('c:\\',)) - check(('/',), '', '\\', ('\\',)) - check(('c:a',), 'c:', '', ('c:', 'a')) - check(('c:/a',), 'c:', '\\', ('c:\\', 'a')) - check(('/a',), '', '\\', ('\\', 'a')) - # UNC paths. - check(('//',), '\\\\', '', ('\\\\',)) - check(('//a',), '\\\\a', '', ('\\\\a',)) - check(('//a/',), '\\\\a\\', '', ('\\\\a\\',)) - check(('//a/b',), '\\\\a\\b', '\\', ('\\\\a\\b\\',)) - check(('//a/b/',), '\\\\a\\b', '\\', ('\\\\a\\b\\',)) - check(('//a/b/c',), '\\\\a\\b', '\\', ('\\\\a\\b\\', 'c')) - # Second part is anchored, so that the first part is ignored. - check(('a', 'Z:b', 'c'), 'Z:', '', ('Z:', 'b', 'c')) - check(('a', 'Z:/b', 'c'), 'Z:', '\\', ('Z:\\', 'b', 'c')) - # UNC paths. - check(('a', '//b/c', 'd'), '\\\\b\\c', '\\', ('\\\\b\\c\\', 'd')) - # Collapsing and stripping excess slashes. - check(('a', 'Z://b//c/', 'd/'), 'Z:', '\\', ('Z:\\', 'b', 'c', 'd')) - # UNC paths. - check(('a', '//b/c//', 'd'), '\\\\b\\c', '\\', ('\\\\b\\c\\', 'd')) - # Extended paths. - check(('//./c:',), '\\\\.\\c:', '', ('\\\\.\\c:',)) - check(('//?/c:/',), '\\\\?\\c:', '\\', ('\\\\?\\c:\\',)) - check(('//?/c:/a',), '\\\\?\\c:', '\\', ('\\\\?\\c:\\', 'a')) - check(('//?/c:/a', '/b'), '\\\\?\\c:', '\\', ('\\\\?\\c:\\', 'b')) - # Extended UNC paths (format is "\\?\UNC\server\share"). - check(('//?',), '\\\\?', '', ('\\\\?',)) - check(('//?/',), '\\\\?\\', '', ('\\\\?\\',)) - check(('//?/UNC',), '\\\\?\\UNC', '', ('\\\\?\\UNC',)) - check(('//?/UNC/',), '\\\\?\\UNC\\', '', ('\\\\?\\UNC\\',)) - check(('//?/UNC/b',), '\\\\?\\UNC\\b', '', ('\\\\?\\UNC\\b',)) - check(('//?/UNC/b/',), '\\\\?\\UNC\\b\\', '', ('\\\\?\\UNC\\b\\',)) - check(('//?/UNC/b/c',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\',)) - check(('//?/UNC/b/c/',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\',)) - check(('//?/UNC/b/c/d',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\', 'd')) - # UNC device paths - check(('//./BootPartition/',), '\\\\.\\BootPartition', '\\', ('\\\\.\\BootPartition\\',)) - check(('//?/BootPartition/',), '\\\\?\\BootPartition', '\\', ('\\\\?\\BootPartition\\',)) - check(('//./PhysicalDrive0',), '\\\\.\\PhysicalDrive0', '', ('\\\\.\\PhysicalDrive0',)) - check(('//?/Volume{}/',), '\\\\?\\Volume{}', '\\', ('\\\\?\\Volume{}\\',)) - check(('//./nul',), '\\\\.\\nul', '', ('\\\\.\\nul',)) - # Second part has a root but not drive. - check(('a', '/b', 'c'), '', '\\', ('\\', 'b', 'c')) - check(('Z:/a', '/b', 'c'), 'Z:', '\\', ('Z:\\', 'b', 'c')) - check(('//?/Z:/a', '/b', 'c'), '\\\\?\\Z:', '\\', ('\\\\?\\Z:\\', 'b', 'c')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - check(('c:/a/b', 'c:x/y'), 'c:', '\\', ('c:\\', 'a', 'b', 'x', 'y')) - check(('c:/a/b', 'c:/x/y'), 'c:', '\\', ('c:\\', 'x', 'y')) - # Paths to files with NTFS alternate data streams - check(('./c:s',), '', '', ('c:s',)) - check(('cc:s',), '', '', ('cc:s',)) - check(('C:c:s',), 'C:', '', ('C:', 'c:s')) - check(('C:/c:s',), 'C:', '\\', ('C:\\', 'c:s')) - check(('D:a', './c:b'), 'D:', '', ('D:', 'a', 'c:b')) - check(('D:/a', './c:b'), 'D:', '\\', ('D:\\', 'a', 'c:b')) - - def test_str(self): - p = self.cls('a/b/c') - self.assertEqual(str(p), 'a\\b\\c') - p = self.cls('c:/a/b/c') - self.assertEqual(str(p), 'c:\\a\\b\\c') - p = self.cls('//a/b') - self.assertEqual(str(p), '\\\\a\\b\\') - p = self.cls('//a/b/c') - self.assertEqual(str(p), '\\\\a\\b\\c') - p = self.cls('//a/b/c/d') - self.assertEqual(str(p), '\\\\a\\b\\c\\d') - - def test_str_subclass(self): - self._check_str_subclass('.\\a:b') - self._check_str_subclass('c:') - self._check_str_subclass('c:a') - self._check_str_subclass('c:a\\b.txt') - self._check_str_subclass('c:\\') - self._check_str_subclass('c:\\a') - self._check_str_subclass('c:\\a\\b.txt') - self._check_str_subclass('\\\\some\\share') - self._check_str_subclass('\\\\some\\share\\a') - self._check_str_subclass('\\\\some\\share\\a\\b.txt') - - def test_eq(self): - P = self.cls - self.assertEqual(P('c:a/b'), P('c:a/b')) - self.assertEqual(P('c:a/b'), P('c:', 'a', 'b')) - self.assertNotEqual(P('c:a/b'), P('d:a/b')) - self.assertNotEqual(P('c:a/b'), P('c:/a/b')) - self.assertNotEqual(P('/a/b'), P('c:/a/b')) - # Case-insensitivity. - self.assertEqual(P('a/B'), P('A/b')) - self.assertEqual(P('C:a/B'), P('c:A/b')) - self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b')) - self.assertEqual(P('\u0130'), P('i\u0307')) - - def test_as_uri(self): - P = self.cls - with self.assertRaises(ValueError): - P('/a/b').as_uri() - with self.assertRaises(ValueError): - P('c:a/b').as_uri() - self.assertEqual(P('c:/').as_uri(), 'file:///c:/') - self.assertEqual(P('c:/a/b.c').as_uri(), 'file:///c:/a/b.c') - self.assertEqual(P('c:/a/b%#c').as_uri(), 'file:///c:/a/b%25%23c') - self.assertEqual(P('c:/a/b\xe9').as_uri(), 'file:///c:/a/b%C3%A9') - self.assertEqual(P('//some/share/').as_uri(), 'file://some/share/') - self.assertEqual(P('//some/share/a/b.c').as_uri(), - 'file://some/share/a/b.c') - self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(), - 'file://some/share/a/b%25%23c%C3%A9') - - def test_match(self): - P = self.cls - # Absolute patterns. - self.assertTrue(P('c:/b.py').match('*:/*.py')) - self.assertTrue(P('c:/b.py').match('c:/*.py')) - self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('b.py').match('c:*.py')) - self.assertFalse(P('b.py').match('c:/*.py')) - self.assertFalse(P('c:b.py').match('/*.py')) - self.assertFalse(P('c:b.py').match('c:/*.py')) - self.assertFalse(P('/b.py').match('c:*.py')) - self.assertFalse(P('/b.py').match('c:/*.py')) - # UNC patterns. - self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) - self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) - # Case-insensitivity. - self.assertTrue(P('B.py').match('b.PY')) - self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) - self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) - # Path anchor doesn't match pattern anchor - self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' - self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' - self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' - - def test_ordering_common(self): - # Case-insensitivity. - def assertOrderedEqual(a, b): - self.assertLessEqual(a, b) - self.assertGreaterEqual(b, a) - P = self.cls - p = P('c:A/b') - q = P('C:a/B') - assertOrderedEqual(p, q) - self.assertFalse(p < q) - self.assertFalse(p > q) - p = P('//some/Share/A/b') - q = P('//Some/SHARE/a/B') - assertOrderedEqual(p, q) - self.assertFalse(p < q) - self.assertFalse(p > q) - - def test_parts(self): - P = self.cls - p = P('c:a/b') - parts = p.parts - self.assertEqual(parts, ('c:', 'a', 'b')) - p = P('c:/a/b') - parts = p.parts - self.assertEqual(parts, ('c:\\', 'a', 'b')) - p = P('//a/b/c/d') - parts = p.parts - self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) - - def test_parent(self): - # Anchored - P = self.cls - p = P('z:a/b/c') - self.assertEqual(p.parent, P('z:a/b')) - self.assertEqual(p.parent.parent, P('z:a')) - self.assertEqual(p.parent.parent.parent, P('z:')) - self.assertEqual(p.parent.parent.parent.parent, P('z:')) - p = P('z:/a/b/c') - self.assertEqual(p.parent, P('z:/a/b')) - self.assertEqual(p.parent.parent, P('z:/a')) - self.assertEqual(p.parent.parent.parent, P('z:/')) - self.assertEqual(p.parent.parent.parent.parent, P('z:/')) - p = P('//a/b/c/d') - self.assertEqual(p.parent, P('//a/b/c')) - self.assertEqual(p.parent.parent, P('//a/b')) - self.assertEqual(p.parent.parent.parent, P('//a/b')) - - def test_parents(self): - # Anchored - P = self.cls - p = P('z:a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:a')) - self.assertEqual(par[1], P('z:')) - self.assertEqual(par[0:1], (P('z:a'),)) - self.assertEqual(par[:-1], (P('z:a'),)) - self.assertEqual(par[:2], (P('z:a'), P('z:'))) - self.assertEqual(par[1:], (P('z:'),)) - self.assertEqual(par[::2], (P('z:a'),)) - self.assertEqual(par[::-1], (P('z:'), P('z:a'))) - self.assertEqual(list(par), [P('z:a'), P('z:')]) - with self.assertRaises(IndexError): - par[2] - p = P('z:/a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:/a')) - self.assertEqual(par[1], P('z:/')) - self.assertEqual(par[0:1], (P('z:/a'),)) - self.assertEqual(par[0:-1], (P('z:/a'),)) - self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) - self.assertEqual(par[1:], (P('z:/'),)) - self.assertEqual(par[::2], (P('z:/a'),)) - self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) - self.assertEqual(list(par), [P('z:/a'), P('z:/')]) - with self.assertRaises(IndexError): - par[2] - p = P('//a/b/c/d') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('//a/b/c')) - self.assertEqual(par[1], P('//a/b')) - self.assertEqual(par[0:1], (P('//a/b/c'),)) - self.assertEqual(par[0:-1], (P('//a/b/c'),)) - self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) - self.assertEqual(par[1:], (P('//a/b'),)) - self.assertEqual(par[::2], (P('//a/b/c'),)) - self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) - self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) - with self.assertRaises(IndexError): - par[2] - - def test_drive(self): - P = self.cls - self.assertEqual(P('c:').drive, 'c:') - self.assertEqual(P('c:a/b').drive, 'c:') - self.assertEqual(P('c:/').drive, 'c:') - self.assertEqual(P('c:/a/b/').drive, 'c:') - self.assertEqual(P('//a/b').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') - self.assertEqual(P('./c:a').drive, '') - - def test_root(self): - P = self.cls - self.assertEqual(P('c:').root, '') - self.assertEqual(P('c:a/b').root, '') - self.assertEqual(P('c:/').root, '\\') - self.assertEqual(P('c:/a/b/').root, '\\') - self.assertEqual(P('//a/b').root, '\\') - self.assertEqual(P('//a/b/').root, '\\') - self.assertEqual(P('//a/b/c/d').root, '\\') - - def test_anchor(self): - P = self.cls - self.assertEqual(P('c:').anchor, 'c:') - self.assertEqual(P('c:a/b').anchor, 'c:') - self.assertEqual(P('c:/').anchor, 'c:\\') - self.assertEqual(P('c:/a/b/').anchor, 'c:\\') - self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') - - def test_name(self): - P = self.cls - self.assertEqual(P('c:').name, '') - self.assertEqual(P('c:/').name, '') - self.assertEqual(P('c:a/b').name, 'b') - self.assertEqual(P('c:/a/b').name, 'b') - self.assertEqual(P('c:a/b.py').name, 'b.py') - self.assertEqual(P('c:/a/b.py').name, 'b.py') - self.assertEqual(P('//My.py/Share.php').name, '') - self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') - - def test_suffix(self): - P = self.cls - self.assertEqual(P('c:').suffix, '') - self.assertEqual(P('c:/').suffix, '') - self.assertEqual(P('c:a/b').suffix, '') - self.assertEqual(P('c:/a/b').suffix, '') - self.assertEqual(P('c:a/b.py').suffix, '.py') - self.assertEqual(P('c:/a/b.py').suffix, '.py') - self.assertEqual(P('c:a/.hgrc').suffix, '') - self.assertEqual(P('c:/a/.hgrc').suffix, '') - self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('//My.py/Share.php').suffix, '') - self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') - - def test_suffixes(self): - P = self.cls - self.assertEqual(P('c:').suffixes, []) - self.assertEqual(P('c:/').suffixes, []) - self.assertEqual(P('c:a/b').suffixes, []) - self.assertEqual(P('c:/a/b').suffixes, []) - self.assertEqual(P('c:a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:a/.hgrc').suffixes, []) - self.assertEqual(P('c:/a/.hgrc').suffixes, []) - self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('//My.py/Share.php').suffixes, []) - self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem(self): - P = self.cls - self.assertEqual(P('c:').stem, '') - self.assertEqual(P('c:.').stem, '') - self.assertEqual(P('c:..').stem, '..') - self.assertEqual(P('c:/').stem, '') - self.assertEqual(P('c:a/b').stem, 'b') - self.assertEqual(P('c:a/b.py').stem, 'b') - self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') - self.assertEqual(P('c:a/.hg.rc').stem, '.hg') - self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name(self): - P = self.cls - self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml')) - self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml')) - self.assertRaises(ValueError, P('c:').with_name, 'd.xml') - self.assertRaises(ValueError, P('c:/').with_name, 'd.xml') - self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml') - self.assertEqual(str(P('a').with_name('d:')), '.\\d:') - self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e') - self.assertEqual(P('c:a/b').with_name('d:'), P('c:a/d:')) - self.assertEqual(P('c:a/b').with_name('d:e'), P('c:a/d:e')) - self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share') - - def test_with_stem(self): - P = self.cls - self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) - self.assertRaises(ValueError, P('c:').with_stem, 'd') - self.assertRaises(ValueError, P('c:/').with_stem, 'd') - self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') - self.assertEqual(str(P('a').with_stem('d:')), '.\\d:') - self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e') - self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:')) - self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e')) - self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') - - def test_with_suffix(self): - P = self.cls - self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) - self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') - - def test_relative_to(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) - self.assertEqual(p.relative_to('c:foO'), P('Bar')) - self.assertEqual(p.relative_to('c:foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR')), P()) - self.assertEqual(p.relative_to('c:foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) - self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) - self.assertRaises(ValueError, p.relative_to, '', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) - p = P('C:/Foo/Bar') - self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) - self.assertEqual(p.relative_to('c:/foO'), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) - self.assertEqual(p.relative_to('c:/foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) - self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, 'c:') - self.assertRaises(ValueError, p.relative_to, P('c:')) - self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo')) - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('d:/')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) - self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) - - def test_is_relative_to(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:'))) - self.assertTrue(p.is_relative_to('c:')) - self.assertTrue(p.is_relative_to(P('c:foO'))) - self.assertTrue(p.is_relative_to('c:foO')) - self.assertTrue(p.is_relative_to('c:foO/')) - self.assertTrue(p.is_relative_to(P('c:foO/baR'))) - self.assertTrue(p.is_relative_to('c:foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('Foo'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('C:/Foo'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) - p = P('C:/Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:/'))) - self.assertTrue(p.is_relative_to(P('c:/foO'))) - self.assertTrue(p.is_relative_to('c:/foO/')) - self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) - self.assertTrue(p.is_relative_to('c:/foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to('c:')) - self.assertFalse(p.is_relative_to(P('C:/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo'))) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('d:/'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('//C/Foo'))) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) - self.assertTrue(p.is_relative_to('//sErver/sHare')) - self.assertTrue(p.is_relative_to('//sErver/sHare/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) - - def test_is_absolute(self): - P = self.cls - # Under NT, only paths with both a drive and a root are absolute. - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertFalse(P('/').is_absolute()) - self.assertFalse(P('/a').is_absolute()) - self.assertFalse(P('/a/b/').is_absolute()) - self.assertFalse(P('c:').is_absolute()) - self.assertFalse(P('c:a').is_absolute()) - self.assertFalse(P('c:a/b/').is_absolute()) - self.assertTrue(P('c:/').is_absolute()) - self.assertTrue(P('c:/a').is_absolute()) - self.assertTrue(P('c:/a/b/').is_absolute()) - # UNC paths are absolute by definition. - self.assertTrue(P('//a/b').is_absolute()) - self.assertTrue(P('//a/b/').is_absolute()) - self.assertTrue(P('//a/b/c').is_absolute()) - self.assertTrue(P('//a/b/c/d').is_absolute()) - - def test_join(self): - P = self.cls - p = P('C:/a/b') - pp = p.joinpath('x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - pp = p.joinpath('D:x/y') - self.assertEqual(pp, P('D:x/y')) - pp = p.joinpath('D:/x/y') - self.assertEqual(pp, P('D:/x/y')) - pp = p.joinpath('//host/share/x/y') - self.assertEqual(pp, P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - pp = p.joinpath('c:x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('c:/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - pp = p.joinpath(P('./d:s')) - self.assertEqual(pp, P('C:/a/b/d:s')) - pp = p.joinpath(P('./dd:s')) - self.assertEqual(pp, P('C:/a/b/dd:s')) - pp = p.joinpath(P('E:d:s')) - self.assertEqual(pp, P('E:d:s')) - # Joining onto a UNC path with no root - pp = P('//').joinpath('server') - self.assertEqual(pp, P('//server')) - pp = P('//server').joinpath('share') - self.assertEqual(pp, P('//server/share')) - pp = P('//./BootPartition').joinpath('Windows') - self.assertEqual(pp, P('//./BootPartition/Windows')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('C:/a/b') - self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) - self.assertEqual(p / '/x/y', P('C:/x/y')) - self.assertEqual(p / '/x' / 'y', P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - self.assertEqual(p / 'D:x/y', P('D:x/y')) - self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) - self.assertEqual(p / 'D:/x/y', P('D:/x/y')) - self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) - self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'c:/x/y', P('C:/x/y')) - # Joining with files with NTFS data streams => the filename should - # not be parsed as a drive letter - self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) - self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) - self.assertEqual(p / P('E:d:s'), P('E:d:s')) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - # UNC paths are never reserved. - self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) - # Case-insensitive DOS-device names are reserved. - self.assertIs(True, P('nul').is_reserved()) - self.assertIs(True, P('aux').is_reserved()) - self.assertIs(True, P('prn').is_reserved()) - self.assertIs(True, P('con').is_reserved()) - self.assertIs(True, P('conin$').is_reserved()) - self.assertIs(True, P('conout$').is_reserved()) - # COM/LPT + 1-9 or + superscript 1-3 are reserved. - self.assertIs(True, P('COM1').is_reserved()) - self.assertIs(True, P('LPT9').is_reserved()) - self.assertIs(True, P('com\xb9').is_reserved()) - self.assertIs(True, P('com\xb2').is_reserved()) - self.assertIs(True, P('lpt\xb3').is_reserved()) - # DOS-device name mataching ignores characters after a dot or - # a colon and also ignores trailing spaces. - self.assertIs(True, P('NUL.txt').is_reserved()) - self.assertIs(True, P('PRN ').is_reserved()) - self.assertIs(True, P('AUX .txt').is_reserved()) - self.assertIs(True, P('COM1:bar').is_reserved()) - self.assertIs(True, P('LPT9 :bar').is_reserved()) - # DOS-device names are only matched at the beginning - # of a path component. - self.assertIs(False, P('bar.com9').is_reserved()) - self.assertIs(False, P('bar.lpt9').is_reserved()) - # Only the last path component matters. - self.assertIs(True, P('c:/baz/con/NUL').is_reserved()) - self.assertIs(False, P('c:/NUL/con/baz').is_reserved()) - - -class PurePathSubclassTest(PurePathTest): - class cls(pathlib.PurePath): - pass - - # repr() roundtripping is not supported in custom subclass. - test_repr_roundtrips = None - - -# -# Tests for the virtual classes. -# - -class PathBaseTest(PurePathTest): - cls = pathlib._PathBase - - def test_unsupported_operation(self): - P = self.cls - p = self.cls() - e = pathlib.UnsupportedOperation - self.assertRaises(e, p.stat) - self.assertRaises(e, p.lstat) - self.assertRaises(e, p.exists) - self.assertRaises(e, p.samefile, 'foo') - self.assertRaises(e, p.is_dir) - self.assertRaises(e, p.is_file) - self.assertRaises(e, p.is_mount) - self.assertRaises(e, p.is_symlink) - self.assertRaises(e, p.is_block_device) - self.assertRaises(e, p.is_char_device) - self.assertRaises(e, p.is_fifo) - self.assertRaises(e, p.is_socket) - self.assertRaises(e, p.open) - self.assertRaises(e, p.read_bytes) - self.assertRaises(e, p.read_text) - self.assertRaises(e, p.write_bytes, b'foo') - self.assertRaises(e, p.write_text, 'foo') - self.assertRaises(e, p.iterdir) - self.assertRaises(e, p.glob, '*') - self.assertRaises(e, p.rglob, '*') - self.assertRaises(e, lambda: list(p.walk())) - self.assertRaises(e, p.absolute) - self.assertRaises(e, P.cwd) - self.assertRaises(e, p.expanduser) - self.assertRaises(e, p.home) - self.assertRaises(e, p.readlink) - self.assertRaises(e, p.symlink_to, 'foo') - self.assertRaises(e, p.hardlink_to, 'foo') - self.assertRaises(e, p.mkdir) - self.assertRaises(e, p.touch) - self.assertRaises(e, p.rename, 'foo') - self.assertRaises(e, p.replace, 'foo') - self.assertRaises(e, p.chmod, 0o755) - self.assertRaises(e, p.lchmod, 0o755) - self.assertRaises(e, p.unlink) - self.assertRaises(e, p.rmdir) - self.assertRaises(e, p.owner) - self.assertRaises(e, p.group) - self.assertRaises(e, p.as_uri) - - def test_as_uri_common(self): - e = pathlib.UnsupportedOperation - self.assertRaises(e, self.cls().as_uri) - - def test_fspath_common(self): - self.assertRaises(TypeError, os.fspath, self.cls()) - - def test_as_bytes_common(self): - self.assertRaises(TypeError, bytes, self.cls()) - - def test_matches_path_api(self): - our_names = {name for name in dir(self.cls) if name[0] != '_'} - path_names = {name for name in dir(pathlib.Path) if name[0] != '_'} - self.assertEqual(our_names, path_names) - for attr_name in our_names: - our_attr = getattr(self.cls, attr_name) - path_attr = getattr(pathlib.Path, attr_name) - self.assertEqual(our_attr.__doc__, path_attr.__doc__) - - -class DummyPathIO(io.BytesIO): - """ - Used by DummyPath to implement `open('w')` - """ - - def __init__(self, files, path): - super().__init__() - self.files = files - self.path = path - - def close(self): - self.files[self.path] = self.getvalue() - super().close() - - -class DummyPath(pathlib._PathBase): - """ - Simple implementation of PathBase that keeps files and directories in - memory. - """ - _files = {} - _directories = {} - _symlinks = {} - - def stat(self, *, follow_symlinks=True): - if follow_symlinks: - path = str(self.resolve()) - else: - path = str(self.parent.resolve() / self.name) - if path in self._files: - st_mode = stat.S_IFREG - elif path in self._directories: - st_mode = stat.S_IFDIR - elif path in self._symlinks: - st_mode = stat.S_IFLNK - else: - raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) - return os.stat_result((st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0)) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - if buffering != -1: - raise NotImplementedError - path_obj = self.resolve() - path = str(path_obj) - name = path_obj.name - parent = str(path_obj.parent) - if path in self._directories: - raise IsADirectoryError(errno.EISDIR, "Is a directory", path) - - text = 'b' not in mode - mode = ''.join(c for c in mode if c not in 'btU') - if mode == 'r': - if path not in self._files: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - stream = io.BytesIO(self._files[path]) - elif mode == 'w': - if parent not in self._directories: - raise FileNotFoundError(errno.ENOENT, "File not found", parent) - stream = DummyPathIO(self._files, path) - self._files[path] = b'' - self._directories[parent].add(name) - else: - raise NotImplementedError - if text: - stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) - return stream - - def iterdir(self): - path = str(self.resolve()) - if path in self._files: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) - elif path in self._directories: - return (self / name for name in self._directories[path]) - else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - path = str(self.resolve()) - if path in self._directories: - if exist_ok: - return - else: - raise FileExistsError(errno.EEXIST, "File exists", path) - try: - if self.name: - self._directories[str(self.parent)].add(self.name) - self._directories[path] = set() - except KeyError: - if not parents: - raise FileNotFoundError(errno.ENOENT, "File not found", str(self.parent)) from None - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - - -class DummyPathTest(unittest.TestCase): - """Tests for PathBase methods that use stat(), open() and iterdir().""" - - cls = DummyPath - can_symlink = False - - # (BASE) - # | - # |-- brokenLink -> non-existing - # |-- dirA - # | `-- linkC -> ../dirB - # |-- dirB - # | |-- fileB - # | `-- linkD -> ../dirB - # |-- dirC - # | |-- dirD - # | | `-- fileD - # | `-- fileC - # | `-- novel.txt - # |-- dirE # No permissions - # |-- fileA - # |-- linkA -> fileA - # |-- linkB -> dirB - # `-- brokenLinkLoop -> brokenLinkLoop - # - - def setUp(self): - super().setUp() - pathmod = self.cls.pathmod - p = self.cls(BASE) - p.mkdir(parents=True) - p.joinpath('dirA').mkdir() - p.joinpath('dirB').mkdir() - p.joinpath('dirC').mkdir() - p.joinpath('dirC', 'dirD').mkdir() - p.joinpath('dirE').mkdir() - with p.joinpath('fileA').open('wb') as f: - f.write(b"this is file A\n") - with p.joinpath('dirB', 'fileB').open('wb') as f: - f.write(b"this is file B\n") - with p.joinpath('dirC', 'fileC').open('wb') as f: - f.write(b"this is file C\n") - with p.joinpath('dirC', 'novel.txt').open('wb') as f: - f.write(b"this is a novel\n") - with p.joinpath('dirC', 'dirD', 'fileD').open('wb') as f: - f.write(b"this is file D\n") - if self.can_symlink: - p.joinpath('linkA').symlink_to('fileA') - p.joinpath('brokenLink').symlink_to('non-existing') - p.joinpath('linkB').symlink_to('dirB') - p.joinpath('dirA', 'linkC').symlink_to(pathmod.join('..', 'dirB')) - p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB')) - p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') - - def tearDown(self): - cls = self.cls - cls._files.clear() - cls._directories.clear() - cls._symlinks.clear() - - def tempdir(self): - path = self.cls(BASE).with_name('tmp-dirD') - path.mkdir() - return path - - def assertFileNotFound(self, func, *args, **kwargs): - with self.assertRaises(FileNotFoundError) as cm: - func(*args, **kwargs) - self.assertEqual(cm.exception.errno, errno.ENOENT) - - def assertEqualNormCase(self, path_a, path_b): - self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b)) - - def test_samefile(self): - fileA_path = os.path.join(BASE, 'fileA') - fileB_path = os.path.join(BASE, 'dirB', 'fileB') - p = self.cls(fileA_path) - pp = self.cls(fileA_path) - q = self.cls(fileB_path) - self.assertTrue(p.samefile(fileA_path)) - self.assertTrue(p.samefile(pp)) - self.assertFalse(p.samefile(fileB_path)) - self.assertFalse(p.samefile(q)) - # Test the non-existent file case - non_existent = os.path.join(BASE, 'foo') - r = self.cls(non_existent) - self.assertRaises(FileNotFoundError, p.samefile, r) - self.assertRaises(FileNotFoundError, p.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, p) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, r) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - - def test_empty_path(self): - # The empty path points to '.' - p = self.cls('') - self.assertEqual(str(p), '.') - - def test_exists(self): - P = self.cls - p = P(BASE) - self.assertIs(True, p.exists()) - self.assertIs(True, (p / 'dirA').exists()) - self.assertIs(True, (p / 'fileA').exists()) - self.assertIs(False, (p / 'fileA' / 'bah').exists()) - if self.can_symlink: - self.assertIs(True, (p / 'linkA').exists()) - self.assertIs(True, (p / 'linkB').exists()) - self.assertIs(True, (p / 'linkB' / 'fileB').exists()) - self.assertIs(False, (p / 'linkA' / 'bah').exists()) - self.assertIs(False, (p / 'brokenLink').exists()) - self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) - self.assertIs(False, (p / 'foo').exists()) - self.assertIs(False, P('/xyzzy').exists()) - self.assertIs(False, P(BASE + '\udfff').exists()) - self.assertIs(False, P(BASE + '\x00').exists()) - - def test_open_common(self): - p = self.cls(BASE) - with (p / 'fileA').open('r') as f: - self.assertIsInstance(f, io.TextIOBase) - self.assertEqual(f.read(), "this is file A\n") - with (p / 'fileA').open('rb') as f: - self.assertIsInstance(f, io.BufferedIOBase) - self.assertEqual(f.read().strip(), b"this is file A") - - def test_read_write_bytes(self): - p = self.cls(BASE) - (p / 'fileA').write_bytes(b'abcdefg') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - # Check that trying to write str does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_bytes, 'somestr') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - - def test_read_write_text(self): - p = self.cls(BASE) - (p / 'fileA').write_text('äbcdefg', encoding='latin-1') - self.assertEqual((p / 'fileA').read_text( - encoding='utf-8', errors='ignore'), 'bcdefg') - # Check that trying to write bytes does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_text, b'somebytes') - self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') - - def test_read_text_with_newlines(self): - p = self.cls(BASE) - # Check that `\n` character change nothing - (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_text(newline='\n'), - 'abcde\r\nfghlk\n\rmnopq') - # Check that `\r` character replaces `\n` - (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_text(newline='\r'), - 'abcde\r\nfghlk\n\rmnopq') - # Check that `\r\n` character replaces `\n` - (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_text(newline='\r\n'), - 'abcde\r\nfghlk\n\rmnopq') - - def test_write_text_with_newlines(self): - p = self.cls(BASE) - # Check that `\n` character change nothing - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\nfghlk\n\rmnopq') - # Check that `\r` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\rfghlk\r\rmnopq') - # Check that `\r\n` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\r\nfghlk\r\n\rmnopq') - # Check that no argument passed will change `\n` to `os.linesep` - os_linesep_byte = bytes(os.linesep, encoding='ascii') - (p / 'fileA').write_text('abcde\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') - - def test_iterdir(self): - P = self.cls - p = P(BASE) - it = p.iterdir() - paths = set(it) - expected = ['dirA', 'dirB', 'dirC', 'dirE', 'fileA'] - if self.can_symlink: - expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] - self.assertEqual(paths, { P(BASE, q) for q in expected }) - - def test_iterdir_symlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") - # __iter__ on a symlink to a directory. - P = self.cls - p = P(BASE, 'linkB') - paths = set(p.iterdir()) - expected = { P(BASE, 'linkB', q) for q in ['fileB', 'linkD'] } - self.assertEqual(paths, expected) - - def test_iterdir_nodir(self): - # __iter__ on something that is not a directory. - p = self.cls(BASE, 'fileA') - with self.assertRaises(OSError) as cm: - p.iterdir() - # ENOENT or EINVAL under Windows, ENOTDIR otherwise - # (see issue #12802). - self.assertIn(cm.exception.errno, (errno.ENOTDIR, - errno.ENOENT, errno.EINVAL)) - - def test_glob_common(self): - def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - it = p.glob("fileA") - self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.glob("fileB"), []) - _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"]) - if not self.can_symlink: - _check(p.glob("*A"), ['dirA', 'fileA']) - else: - _check(p.glob("*A"), ['dirA', 'fileA', 'linkA']) - if not self.can_symlink: - _check(p.glob("*B/*"), ['dirB/fileB']) - else: - _check(p.glob("*B/*"), ['dirB/fileB', 'dirB/linkD', - 'linkB/fileB', 'linkB/linkD']) - if not self.can_symlink: - _check(p.glob("*/fileB"), ['dirB/fileB']) - else: - _check(p.glob("*/fileB"), ['dirB/fileB', 'linkB/fileB']) - if self.can_symlink: - _check(p.glob("brokenLink"), ['brokenLink']) - - if not self.can_symlink: - _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE"]) - else: - _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE", "linkB"]) - - def test_glob_empty_pattern(self): - p = self.cls() - with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): - list(p.glob('')) - - def test_glob_case_sensitive(self): - P = self.cls - def _check(path, pattern, case_sensitive, expected): - actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} - expected = {str(P(BASE, q)) for q in expected} - self.assertEqual(actual, expected) - path = P(BASE) - _check(path, "DIRB/FILE*", True, []) - _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) - _check(path, "dirb/file*", True, []) - _check(path, "dirb/file*", False, ["dirB/fileB"]) - - def test_glob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=True) - if "linkD" not in path.parent.parts} # exclude symlink loop. - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) - _check(p, "*/", ["dirA", "dirB", "dirC", "dirE", "linkB"]) - _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/.."]) - _check(p, "dir*/**/", ["dirA", "dirA/linkC", "dirA/linkC/linkD", "dirB", "dirB/linkD", - "dirC", "dirC/dirD", "dirE"]) - _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", - "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**/", ["dirA/linkC", "dirA/linkC/linkD", "dirB/linkD", "dirC/dirD"]) - _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD"]) - _check(p, "*/dirD/**/", ["dirC/dirD"]) - - def test_glob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "*/", ["dirA", "dirB", "dirC", "dirE"]) - _check(p, "dir*/*/..", ["dirC/dirD/.."]) - _check(p, "dir*/**/", ["dirA", "dirB", "dirC", "dirC/dirD", "dirE"]) - _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**/", ["dirC/dirD"]) - _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD"]) - _check(p, "*/dirD/**/", ["dirC/dirD"]) - - def test_rglob_common(self): - def _check(glob, expected): - self.assertEqual(sorted(glob), sorted(P(BASE, q) for q in expected)) - P = self.cls - p = P(BASE) - it = p.rglob("fileA") - self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.rglob("fileB"), ["dirB/fileB"]) - _check(p.rglob("**/fileB"), ["dirB/fileB"]) - _check(p.rglob("*/fileA"), []) - if not self.can_symlink: - _check(p.rglob("*/fileB"), ["dirB/fileB"]) - else: - _check(p.rglob("*/fileB"), ["dirB/fileB", "dirB/linkD/fileB", - "linkB/fileB", "dirA/linkC/fileB"]) - _check(p.rglob("file*"), ["fileA", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD"]) - if not self.can_symlink: - _check(p.rglob("*/"), [ - "dirA", "dirB", "dirC", "dirC/dirD", "dirE", - ]) - else: - _check(p.rglob("*/"), [ - "dirA", "dirA/linkC", "dirB", "dirB/linkD", "dirC", - "dirC/dirD", "dirE", "linkB", - ]) - _check(p.rglob(""), ["", "dirA", "dirB", "dirC", "dirE", "dirC/dirD"]) - - p = P(BASE, "dirC") - _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**/"), ["dirC/dirD"]) - _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) - _check(p.rglob("*/"), ["dirC/dirD"]) - _check(p.rglob(""), ["dirC", "dirC/dirD"]) - _check(p.rglob("**/"), ["dirC", "dirC/dirD"]) - # gh-91616, a re module regression - _check(p.rglob("*.txt"), ["dirC/novel.txt"]) - _check(p.rglob("*.*"), ["dirC/novel.txt"]) - - def test_rglob_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=True) - if 'linkD' not in path.parent.parts} # exclude symlink loop. - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) - _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) - _check(p, "*/", ["dirA", "dirA/linkC", "dirA/linkC/linkD", "dirB", "dirB/linkD", - "dirC", "dirC/dirD", "dirE", "linkB", "linkB/linkD"]) - _check(p, "", ["", "dirA", "dirA/linkC", "dirA/linkC/linkD", "dirB", "dirB/linkD", - "dirC", "dirE", "dirC/dirD", "linkB", "linkB/linkD"]) - - p = P(BASE, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD"]) - _check(p, "", ["dirC", "dirC/dirD"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - - def test_rglob_no_follow_symlinks_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - _check(p, "fileB", ["dirB/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", ]) - _check(p, "*/", ["dirA", "dirB", "dirC", "dirC/dirD", "dirE"]) - _check(p, "", ["", "dirA", "dirB", "dirC", "dirE", "dirC/dirD"]) - - p = P(BASE, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD"]) - _check(p, "", ["dirC", "dirC/dirD"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - - def test_rglob_symlink_loop(self): - # Don't get fooled by symlink loops (Issue #26012). - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls - p = P(BASE) - given = set(p.rglob('*')) - expect = {'brokenLink', - 'dirA', 'dirA/linkC', - 'dirB', 'dirB/fileB', 'dirB/linkD', - 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', - 'dirC/fileC', 'dirC/novel.txt', - 'dirE', - 'fileA', - 'linkA', - 'linkB', - 'brokenLinkLoop', - } - self.assertEqual(given, {p / x for x in expect}) - - def test_glob_many_open_files(self): - depth = 30 - P = self.cls - p = base = P(BASE) / 'deep' - p.mkdir() - for _ in range(depth): - p /= 'd' - p.mkdir() - pattern = '/'.join(['*'] * depth) - iters = [base.glob(pattern) for j in range(100)] - for it in iters: - self.assertEqual(next(it), p) - iters = [base.rglob('d') for j in range(100)] - p = base - for i in range(depth): - p = p / 'd' - for it in iters: - self.assertEqual(next(it), p) - - def test_glob_dotdot(self): - # ".." is not special in globs. - P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("..")), { P(BASE, "..") }) - self.assertEqual(set(p.glob("../..")), { P(BASE, "..", "..") }) - self.assertEqual(set(p.glob("dirA/..")), { P(BASE, "dirA", "..") }) - self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) - self.assertEqual(set(p.glob("dirA/../file*/..")), set()) - self.assertEqual(set(p.glob("../xyzzy")), set()) - self.assertEqual(set(p.glob("xyzzy/..")), set()) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(BASE, *[".."] * 50)}) - - def test_glob_permissions(self): - # See bpo-38894 - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls - base = P(BASE) / 'permissions' - base.mkdir() - - for i in range(100): - link = base / f"link{i}" - if i % 2: - link.symlink_to(P(BASE, "dirE", "nonexistent")) - else: - link.symlink_to(P(BASE, "dirC")) - - self.assertEqual(len(set(base.glob("*"))), 100) - self.assertEqual(len(set(base.glob("*/"))), 50) - self.assertEqual(len(set(base.glob("*/fileC"))), 50) - self.assertEqual(len(set(base.glob("*/file*"))), 50) - - def test_glob_long_symlink(self): - # See gh-87695 - if not self.can_symlink: - self.skipTest("symlinks required") - base = self.cls(BASE) / 'long_symlink' - base.mkdir() - bad_link = base / 'bad_link' - bad_link.symlink_to("bad" * 200) - self.assertEqual(sorted(base.glob('**/*')), [bad_link]) - - def test_glob_above_recursion_limit(self): - recursion_limit = 50 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = self.cls(BASE, 'deep') - path = self.cls(base, *(['d'] * directory_depth)) - path.mkdir(parents=True) - - with set_recursion_limit(recursion_limit): - list(base.glob('**/')) - - def test_glob_recursive_no_trailing_slash(self): - P = self.cls - p = P(BASE) - with self.assertWarns(FutureWarning): - p.glob('**') - with self.assertWarns(FutureWarning): - p.glob('*/**') - with self.assertWarns(FutureWarning): - p.rglob('**') - with self.assertWarns(FutureWarning): - p.rglob('*/**') - - - def test_readlink(self): - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls(BASE) - self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) - self.assertEqual((P / 'brokenLink').readlink(), - self.cls('non-existing')) - self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) - self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) - with self.assertRaises(OSError): - (P / 'fileA').readlink() - - @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") - def test_readlink_unsupported(self): - P = self.cls(BASE) - p = P / 'fileA' - with self.assertRaises(pathlib.UnsupportedOperation): - q.readlink(p) - - def _check_resolve(self, p, expected, strict=True): - q = p.resolve(strict) - self.assertEqual(q, expected) - - # This can be used to check both relative and absolute resolutions. - _check_resolve_relative = _check_resolve_absolute = _check_resolve - - def test_resolve_common(self): - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls - p = P(BASE, 'foo') - with self.assertRaises(OSError) as cm: - p.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ENOENT) - # Non-strict - self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) - p = P(BASE, 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo', 'in', 'spam')) - p = P(BASE, '..', 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo', 'in', 'spam'))) - # These are all relative symlinks. - p = P(BASE, 'dirB', 'fileB') - self._check_resolve_relative(p, p) - p = P(BASE, 'linkA') - self._check_resolve_relative(p, P(BASE, 'fileA')) - p = P(BASE, 'dirA', 'linkC', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) - p = P(BASE, 'dirB', 'linkD', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) - # Non-strict - p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', - 'spam'), False) - p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if os.name == 'nt' and isinstance(p, pathlib.Path): - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', - 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) - # Now create absolute symlinks. - d = self.tempdir() - P(BASE, 'dirA', 'linkX').symlink_to(d) - P(BASE, str(d), 'linkY').symlink_to(join('dirB')) - p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') - self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) - # Non-strict - p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), - False) - p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if os.name == 'nt' and isinstance(p, pathlib.Path): - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) - - def test_resolve_dot(self): - # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - if not self.can_symlink: - self.skipTest("symlinks required") - p = self.cls(BASE) - p.joinpath('0').symlink_to('.', target_is_directory=True) - p.joinpath('1').symlink_to(os.path.join('0', '0'), target_is_directory=True) - p.joinpath('2').symlink_to(os.path.join('1', '1'), target_is_directory=True) - q = p / '2' - self.assertEqual(q.resolve(strict=True), p) - r = q / '3' / '4' - self.assertRaises(FileNotFoundError, r.resolve, strict=True) - # Non-strict - self.assertEqual(r.resolve(strict=False), p / '3' / '4') - - def _check_symlink_loop(self, *args): - path = self.cls(*args) - with self.assertRaises(OSError) as cm: - path.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ELOOP) - - def test_resolve_loop(self): - if not self.can_symlink: - self.skipTest("symlinks required") - if os.name == 'nt' and issubclass(self.cls, pathlib.Path): - self.skipTest("symlink loops work differently with concrete Windows paths") - # Loops with relative symlinks. - self.cls(BASE, 'linkX').symlink_to('linkX/inside') - self._check_symlink_loop(BASE, 'linkX') - self.cls(BASE, 'linkY').symlink_to('linkY') - self._check_symlink_loop(BASE, 'linkY') - self.cls(BASE, 'linkZ').symlink_to('linkZ/../linkZ') - self._check_symlink_loop(BASE, 'linkZ') - # Non-strict - p = self.cls(BASE, 'linkZ', 'foo') - self.assertEqual(p.resolve(strict=False), p) - # Loops with absolute symlinks. - self.cls(BASE, 'linkU').symlink_to(join('linkU/inside')) - self._check_symlink_loop(BASE, 'linkU') - self.cls(BASE, 'linkV').symlink_to(join('linkV')) - self._check_symlink_loop(BASE, 'linkV') - self.cls(BASE, 'linkW').symlink_to(join('linkW/../linkW')) - self._check_symlink_loop(BASE, 'linkW') - # Non-strict - q = self.cls(BASE, 'linkW', 'foo') - self.assertEqual(q.resolve(strict=False), q) - - def test_stat(self): - statA = self.cls(BASE).joinpath('fileA').stat() - statB = self.cls(BASE).joinpath('dirB', 'fileB').stat() - statC = self.cls(BASE).joinpath('dirC').stat() - # st_mode: files are the same, directory differs. - self.assertIsInstance(statA.st_mode, int) - self.assertEqual(statA.st_mode, statB.st_mode) - self.assertNotEqual(statA.st_mode, statC.st_mode) - self.assertNotEqual(statB.st_mode, statC.st_mode) - # st_ino: all different, - self.assertIsInstance(statA.st_ino, int) - self.assertNotEqual(statA.st_ino, statB.st_ino) - self.assertNotEqual(statA.st_ino, statC.st_ino) - self.assertNotEqual(statB.st_ino, statC.st_ino) - # st_dev: all the same. - self.assertIsInstance(statA.st_dev, int) - self.assertEqual(statA.st_dev, statB.st_dev) - self.assertEqual(statA.st_dev, statC.st_dev) - # other attributes not used by pathlib. - - def test_stat_no_follow_symlinks(self): - if not self.can_symlink: - self.skipTest("symlinks required") - p = self.cls(BASE) / 'linkA' - st = p.stat() - self.assertNotEqual(st, p.stat(follow_symlinks=False)) - - def test_stat_no_follow_symlinks_nosymlink(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(st, p.stat(follow_symlinks=False)) - - def test_lstat(self): - if not self.can_symlink: - self.skipTest("symlinks required") - p = self.cls(BASE)/ 'linkA' - st = p.stat() - self.assertNotEqual(st, p.lstat()) - - def test_lstat_nosymlink(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(st, p.lstat()) - - def test_is_dir(self): - P = self.cls(BASE) - self.assertTrue((P / 'dirA').is_dir()) - self.assertFalse((P / 'fileA').is_dir()) - self.assertFalse((P / 'non-existing').is_dir()) - self.assertFalse((P / 'fileA' / 'bah').is_dir()) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_dir()) - self.assertTrue((P / 'linkB').is_dir()) - self.assertFalse((P/ 'brokenLink').is_dir()) - self.assertFalse((P / 'dirA\udfff').is_dir()) - self.assertFalse((P / 'dirA\x00').is_dir()) - - def test_is_dir_no_follow_symlinks(self): - P = self.cls(BASE) - self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'fileA' / 'bah').is_dir(follow_symlinks=False)) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'linkB').is_dir(follow_symlinks=False)) - self.assertFalse((P/ 'brokenLink').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'dirA\udfff').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) - - def test_is_file(self): - P = self.cls(BASE) - self.assertTrue((P / 'fileA').is_file()) - self.assertFalse((P / 'dirA').is_file()) - self.assertFalse((P / 'non-existing').is_file()) - self.assertFalse((P / 'fileA' / 'bah').is_file()) - if self.can_symlink: - self.assertTrue((P / 'linkA').is_file()) - self.assertFalse((P / 'linkB').is_file()) - self.assertFalse((P/ 'brokenLink').is_file()) - self.assertFalse((P / 'fileA\udfff').is_file()) - self.assertFalse((P / 'fileA\x00').is_file()) - - def test_is_file_no_follow_symlinks(self): - P = self.cls(BASE) - self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) - self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) - self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) - self.assertFalse((P / 'fileA' / 'bah').is_file(follow_symlinks=False)) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_file(follow_symlinks=False)) - self.assertFalse((P / 'linkB').is_file(follow_symlinks=False)) - self.assertFalse((P/ 'brokenLink').is_file(follow_symlinks=False)) - self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) - self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) - - def test_is_mount(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_mount()) - self.assertFalse((P / 'dirA').is_mount()) - self.assertFalse((P / 'non-existing').is_mount()) - self.assertFalse((P / 'fileA' / 'bah').is_mount()) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_mount()) - - def test_is_symlink(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_symlink()) - self.assertFalse((P / 'dirA').is_symlink()) - self.assertFalse((P / 'non-existing').is_symlink()) - self.assertFalse((P / 'fileA' / 'bah').is_symlink()) - if self.can_symlink: - self.assertTrue((P / 'linkA').is_symlink()) - self.assertTrue((P / 'linkB').is_symlink()) - self.assertTrue((P/ 'brokenLink').is_symlink()) - self.assertIs((P / 'fileA\udfff').is_file(), False) - self.assertIs((P / 'fileA\x00').is_file(), False) - if self.can_symlink: - self.assertIs((P / 'linkA\udfff').is_file(), False) - self.assertIs((P / 'linkA\x00').is_file(), False) - - def test_is_junction_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_junction()) - self.assertFalse((P / 'dirA').is_junction()) - self.assertFalse((P / 'non-existing').is_junction()) - self.assertFalse((P / 'fileA' / 'bah').is_junction()) - self.assertFalse((P / 'fileA\udfff').is_junction()) - self.assertFalse((P / 'fileA\x00').is_junction()) - - def test_is_fifo_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_fifo()) - self.assertFalse((P / 'dirA').is_fifo()) - self.assertFalse((P / 'non-existing').is_fifo()) - self.assertFalse((P / 'fileA' / 'bah').is_fifo()) - self.assertIs((P / 'fileA\udfff').is_fifo(), False) - self.assertIs((P / 'fileA\x00').is_fifo(), False) - - def test_is_socket_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_socket()) - self.assertFalse((P / 'dirA').is_socket()) - self.assertFalse((P / 'non-existing').is_socket()) - self.assertFalse((P / 'fileA' / 'bah').is_socket()) - self.assertIs((P / 'fileA\udfff').is_socket(), False) - self.assertIs((P / 'fileA\x00').is_socket(), False) - - def test_is_block_device_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_block_device()) - self.assertFalse((P / 'dirA').is_block_device()) - self.assertFalse((P / 'non-existing').is_block_device()) - self.assertFalse((P / 'fileA' / 'bah').is_block_device()) - self.assertIs((P / 'fileA\udfff').is_block_device(), False) - self.assertIs((P / 'fileA\x00').is_block_device(), False) - - def test_is_char_device_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_char_device()) - self.assertFalse((P / 'dirA').is_char_device()) - self.assertFalse((P / 'non-existing').is_char_device()) - self.assertFalse((P / 'fileA' / 'bah').is_char_device()) - self.assertIs((P / 'fileA\udfff').is_char_device(), False) - self.assertIs((P / 'fileA\x00').is_char_device(), False) - - def test_pickling_common(self): - p = self.cls(BASE, 'fileA') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertEqual(pp.stat(), p.stat()) - - def test_parts_interning(self): - P = self.cls - p = P('/usr/bin/foo') - q = P('/usr/local/bin') - # 'usr' - self.assertIs(p.parts[1], q.parts[1]) - # 'bin' - self.assertIs(p.parts[2], q.parts[3]) - - def _check_complex_symlinks(self, link0_target): - if not self.can_symlink: - self.skipTest("symlinks required") - - # Test solving a non-looping chain of symlinks (issue #19887). - P = self.cls(BASE) - P.joinpath('link1').symlink_to(os.path.join('link0', 'link0'), target_is_directory=True) - P.joinpath('link2').symlink_to(os.path.join('link1', 'link1'), target_is_directory=True) - P.joinpath('link3').symlink_to(os.path.join('link2', 'link2'), target_is_directory=True) - P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) - - # Resolve absolute paths. - p = (P / 'link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - - # Resolve relative paths. - try: - self.cls().absolute() - except pathlib.UnsupportedOperation: - return - old_path = os.getcwd() - os.chdir(BASE) - try: - p = self.cls('link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - finally: - os.chdir(old_path) - - def test_complex_symlinks_absolute(self): - self._check_complex_symlinks(BASE) - - def test_complex_symlinks_relative(self): - self._check_complex_symlinks('.') - - def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(os.path.join('dirA', '..')) - - def setUpWalk(self): - # Build: - # TESTFN/ - # TEST1/ a file kid and two directory kids - # tmp1 - # SUB1/ a file kid and a directory kid - # tmp2 - # SUB11/ no kids - # SUB2/ a file kid and a dirsymlink kid - # tmp3 - # link/ a symlink to TEST2 - # broken_link - # broken_link2 - # TEST2/ - # tmp4 a lone file - self.walk_path = self.cls(BASE, "TEST1") - self.sub1_path = self.walk_path / "SUB1" - self.sub11_path = self.sub1_path / "SUB11" - self.sub2_path = self.walk_path / "SUB2" - tmp1_path = self.walk_path / "tmp1" - tmp2_path = self.sub1_path / "tmp2" - tmp3_path = self.sub2_path / "tmp3" - self.link_path = self.sub2_path / "link" - t2_path = self.cls(BASE, "TEST2") - tmp4_path = self.cls(BASE, "TEST2", "tmp4") - broken_link_path = self.sub2_path / "broken_link" - broken_link2_path = self.sub2_path / "broken_link2" - - self.sub11_path.mkdir(parents=True) - self.sub2_path.mkdir(parents=True) - t2_path.mkdir(parents=True) - - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: - with path.open("w", encoding='utf-8') as f: - f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") - - if self.can_symlink: - self.link_path.symlink_to(t2_path) - broken_link_path.symlink_to('broken') - broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) - self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) - else: - self.sub2_tree = (self.sub2_path, [], ["tmp3"]) - - def test_walk_topdown(self): - self.setUpWalk() - walker = self.walk_path.walk() - entry = next(walker) - entry[1].sort() # Ensure we visit SUB1 before SUB2 - self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"])) - entry = next(walker) - self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"])) - entry = next(walker) - self.assertEqual(entry, (self.sub11_path, [], [])) - entry = next(walker) - entry[1].sort() - entry[2].sort() - self.assertEqual(entry, self.sub2_tree) - with self.assertRaises(StopIteration): - next(walker) - - def test_walk_prune(self): - self.setUpWalk() - # Prune the search. - all = [] - for root, dirs, files in self.walk_path.walk(): - all.append((root, dirs, files)) - if 'SUB1' in dirs: - # Note that this also mutates the dirs we appended to all! - dirs.remove('SUB1') - - self.assertEqual(len(all), 2) - self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) - - all[1][-1].sort() - all[1][1].sort() - self.assertEqual(all[1], self.sub2_tree) - - def test_walk_bottom_up(self): - self.setUpWalk() - seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False - for path, dirnames, filenames in self.walk_path.walk(top_down=False): - if path == self.walk_path: - self.assertFalse(seen_testfn) - self.assertTrue(seen_sub1) - self.assertTrue(seen_sub2) - self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"]) - self.assertEqual(filenames, ["tmp1"]) - seen_testfn = True - elif path == self.sub1_path: - self.assertFalse(seen_testfn) - self.assertFalse(seen_sub1) - self.assertTrue(seen_sub11) - self.assertEqual(dirnames, ["SUB11"]) - self.assertEqual(filenames, ["tmp2"]) - seen_sub1 = True - elif path == self.sub11_path: - self.assertFalse(seen_sub1) - self.assertFalse(seen_sub11) - self.assertEqual(dirnames, []) - self.assertEqual(filenames, []) - seen_sub11 = True - elif path == self.sub2_path: - self.assertFalse(seen_testfn) - self.assertFalse(seen_sub2) - self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1])) - self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2])) - seen_sub2 = True - else: - raise AssertionError(f"Unexpected path: {path}") - self.assertTrue(seen_testfn) - - def test_walk_follow_symlinks(self): - if not self.can_symlink: - self.skipTest("symlinks required") - self.setUpWalk() - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.link_path: - self.assertEqual(dirs, []) - self.assertEqual(files, ["tmp4"]) - break - else: - self.fail("Didn't follow symlink with follow_symlinks=True") - - def test_walk_symlink_location(self): - if not self.can_symlink: - self.skipTest("symlinks required") - self.setUpWalk() - # Tests whether symlinks end up in filenames or dirnames depending - # on the `follow_symlinks` argument. - walk_it = self.walk_path.walk(follow_symlinks=False) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", files) - break - else: - self.fail("symlink not found") - - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", dirs) - break - else: - self.fail("symlink not found") - - def test_walk_above_recursion_limit(self): - recursion_limit = 40 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = self.cls(BASE, 'deep') - path = self.cls(base, *(['d'] * directory_depth)) - path.mkdir(parents=True) - - with set_recursion_limit(recursion_limit): - list(base.walk()) - list(base.walk(top_down=False)) - -class DummyPathWithSymlinks(DummyPath): - def readlink(self): - path = str(self.parent.resolve() / self.name) - if path in self._symlinks: - return self.with_segments(self._symlinks[path]) - elif path in self._files or path in self._directories: - raise OSError(errno.EINVAL, "Not a symlink", path) - else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def symlink_to(self, target, target_is_directory=False): - self._directories[str(self.parent)].add(self.name) - self._symlinks[str(self)] = str(target) - - -class DummyPathWithSymlinksTest(DummyPathTest): - cls = DummyPathWithSymlinks - can_symlink = True - - -# -# Tests for the concrete classes. -# - -class PathTest(DummyPathTest, PurePathTest): - """Tests for the FS-accessing functionalities of the Path classes.""" - cls = pathlib.Path - can_symlink = os_helper.can_symlink() - - def setUp(self): - super().setUp() - os.chmod(join('dirE'), 0) - - def tearDown(self): - os.chmod(join('dirE'), 0o777) - os_helper.rmtree(BASE) - - def tempdir(self): - d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', - dir=os.getcwd())) - self.addCleanup(os_helper.rmtree, d) - return d - - def test_concrete_class(self): - if self.cls is pathlib.Path: - expected = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath - else: - expected = self.cls - p = self.cls('a') - self.assertIs(type(p), expected) - - def test_unsupported_pathmod(self): - if self.cls.pathmod is os.path: - self.skipTest("path flavour is supported") - else: - self.assertRaises(pathlib.UnsupportedOperation, self.cls) - - def _test_cwd(self, p): - q = self.cls(os.getcwd()) - self.assertEqual(p, q) - self.assertEqualNormCase(str(p), str(q)) - self.assertIs(type(p), type(q)) - self.assertTrue(p.is_absolute()) - - def test_cwd(self): - p = self.cls.cwd() - self._test_cwd(p) - - def test_absolute_common(self): - P = self.cls - - with mock.patch("os.getcwd") as getcwd: - getcwd.return_value = BASE - - # Simple relative paths. - self.assertEqual(str(P().absolute()), BASE) - self.assertEqual(str(P('.').absolute()), BASE) - self.assertEqual(str(P('a').absolute()), os.path.join(BASE, 'a')) - self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(BASE, 'a', 'b', 'c')) - - # Symlinks should not be resolved. - self.assertEqual(str(P('linkB', 'fileB').absolute()), os.path.join(BASE, 'linkB', 'fileB')) - self.assertEqual(str(P('brokenLink').absolute()), os.path.join(BASE, 'brokenLink')) - self.assertEqual(str(P('brokenLinkLoop').absolute()), os.path.join(BASE, 'brokenLinkLoop')) - - # '..' entries should be preserved and not normalised. - self.assertEqual(str(P('..').absolute()), os.path.join(BASE, '..')) - self.assertEqual(str(P('a', '..').absolute()), os.path.join(BASE, 'a', '..')) - self.assertEqual(str(P('..', 'b').absolute()), os.path.join(BASE, '..', 'b')) - - def _test_home(self, p): - q = self.cls(os.path.expanduser('~')) - self.assertEqual(p, q) - self.assertEqualNormCase(str(p), str(q)) - self.assertIs(type(p), type(q)) - self.assertTrue(p.is_absolute()) - - @unittest.skipIf( - pwd is None, reason="Test requires pwd module to get homedir." - ) - def test_home(self): - with os_helper.EnvironmentVarGuard() as env: - self._test_home(self.cls.home()) - - env.clear() - env['USERPROFILE'] = os.path.join(BASE, 'userprofile') - self._test_home(self.cls.home()) - - # bpo-38883: ignore `HOME` when set on windows - env['HOME'] = os.path.join(BASE, 'home') - self._test_home(self.cls.home()) - - @unittest.skipIf(is_wasi, "WASI has no user accounts.") - def test_expanduser_common(self): - P = self.cls - p = P('~') - self.assertEqual(p.expanduser(), P(os.path.expanduser('~'))) - p = P('foo') - self.assertEqual(p.expanduser(), p) - p = P('/~') - self.assertEqual(p.expanduser(), p) - p = P('../~') - self.assertEqual(p.expanduser(), p) - p = P(P('').absolute().anchor) / '~' - self.assertEqual(p.expanduser(), p) - p = P('~/a:b') - self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) - - def test_with_segments(self): - class P(self.cls): - def __init__(self, *pathsegments, session_id): - super().__init__(*pathsegments) - self.session_id = session_id - - def with_segments(self, *pathsegments): - return type(self)(*pathsegments, session_id=self.session_id) - p = P(BASE, session_id=42) - self.assertEqual(42, p.absolute().session_id) - self.assertEqual(42, p.resolve().session_id) - if not is_wasi: # WASI has no user accounts. - self.assertEqual(42, p.with_segments('~').expanduser().session_id) - self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) - self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) - if self.can_symlink: - self.assertEqual(42, (p / 'linkA').readlink().session_id) - for path in p.iterdir(): - self.assertEqual(42, path.session_id) - for path in p.glob('*'): - self.assertEqual(42, path.session_id) - for path in p.rglob('*'): - self.assertEqual(42, path.session_id) - for dirpath, dirnames, filenames in p.walk(): - self.assertEqual(42, dirpath.session_id) - - def test_open_unbuffered(self): - p = self.cls(BASE) - with (p / 'fileA').open('rb', buffering=0) as f: - self.assertIsInstance(f, io.RawIOBase) - self.assertEqual(f.read().strip(), b"this is file A") - - def test_resolve_nonexist_relative_issue38671(self): - p = self.cls('non', 'exist') - - old_cwd = os.getcwd() - os.chdir(BASE) - try: - self.assertEqual(p.resolve(), self.cls(BASE, p)) - finally: - os.chdir(old_cwd) - - @os_helper.skip_unless_working_chmod - def test_chmod(self): - p = self.cls(BASE) / 'fileA' - mode = p.stat().st_mode - # Clear writable bit. - new_mode = mode & ~0o222 - p.chmod(new_mode) - self.assertEqual(p.stat().st_mode, new_mode) - # Set writable bit. - new_mode = mode | 0o222 - p.chmod(new_mode) - self.assertEqual(p.stat().st_mode, new_mode) - - # On Windows, os.chmod does not follow symlinks (issue #15411) - @only_posix - @os_helper.skip_unless_working_chmod - def test_chmod_follow_symlinks_true(self): - p = self.cls(BASE) / 'linkA' - q = p.resolve() - mode = q.stat().st_mode - # Clear writable bit. - new_mode = mode & ~0o222 - p.chmod(new_mode, follow_symlinks=True) - self.assertEqual(q.stat().st_mode, new_mode) - # Set writable bit - new_mode = mode | 0o222 - p.chmod(new_mode, follow_symlinks=True) - self.assertEqual(q.stat().st_mode, new_mode) - - # XXX also need a test for lchmod. - - @unittest.skipUnless(pwd, "the pwd module is needed for this test") - def test_owner(self): - p = self.cls(BASE) / 'fileA' - uid = p.stat().st_uid - try: - name = pwd.getpwuid(uid).pw_name - except KeyError: - self.skipTest( - "user %d doesn't have an entry in the system database" % uid) - self.assertEqual(name, p.owner()) - - @unittest.skipUnless(grp, "the grp module is needed for this test") - def test_group(self): - p = self.cls(BASE) / 'fileA' - gid = p.stat().st_gid - try: - name = grp.getgrgid(gid).gr_name - except KeyError: - self.skipTest( - "group %d doesn't have an entry in the system database" % gid) - self.assertEqual(name, p.group()) - - def test_unlink(self): - p = self.cls(BASE) / 'fileA' - p.unlink() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) - - def test_unlink_missing_ok(self): - p = self.cls(BASE) / 'fileAAA' - self.assertFileNotFound(p.unlink) - p.unlink(missing_ok=True) - - def test_rmdir(self): - p = self.cls(BASE) / 'dirA' - for q in p.iterdir(): - q.unlink() - p.rmdir() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) - - @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") - def test_hardlink_to(self): - P = self.cls(BASE) - target = P / 'fileA' - size = target.stat().st_size - # linking to another path. - link = P / 'dirA' / 'fileAA' - link.hardlink_to(target) - self.assertEqual(link.stat().st_size, size) - self.assertTrue(os.path.samefile(target, link)) - self.assertTrue(target.exists()) - # Linking to a str of a relative path. - link2 = P / 'dirA' / 'fileAAA' - target2 = rel_join('fileA') - link2.hardlink_to(target2) - self.assertEqual(os.stat(target2).st_size, size) - self.assertTrue(link2.exists()) - - @unittest.skipIf(hasattr(os, "link"), "os.link() is present") - def test_hardlink_to_unsupported(self): - P = self.cls(BASE) - p = P / 'fileA' - # linking to another path. - q = P / 'dirA' / 'fileAA' - with self.assertRaises(pathlib.UnsupportedOperation): - q.hardlink_to(p) - - def test_rename(self): - P = self.cls(BASE) - p = P / 'fileA' - size = p.stat().st_size - # Renaming to another path. - q = P / 'dirA' / 'fileAA' - renamed_p = p.rename(q) - self.assertEqual(renamed_p, q) - self.assertEqual(q.stat().st_size, size) - self.assertFileNotFound(p.stat) - # Renaming to a str of a relative path. - r = rel_join('fileAAA') - renamed_q = q.rename(r) - self.assertEqual(renamed_q, self.cls(r)) - self.assertEqual(os.stat(r).st_size, size) - self.assertFileNotFound(q.stat) - - def test_replace(self): - P = self.cls(BASE) - p = P / 'fileA' - size = p.stat().st_size - # Replacing a non-existing path. - q = P / 'dirA' / 'fileAA' - replaced_p = p.replace(q) - self.assertEqual(replaced_p, q) - self.assertEqual(q.stat().st_size, size) - self.assertFileNotFound(p.stat) - # Replacing another (existing) path. - r = rel_join('dirB', 'fileB') - replaced_q = q.replace(r) - self.assertEqual(replaced_q, self.cls(r)) - self.assertEqual(os.stat(r).st_size, size) - self.assertFileNotFound(q.stat) - - def test_touch_common(self): - P = self.cls(BASE) - p = P / 'newfileA' - self.assertFalse(p.exists()) - p.touch() - self.assertTrue(p.exists()) - st = p.stat() - old_mtime = st.st_mtime - old_mtime_ns = st.st_mtime_ns - # Rewind the mtime sufficiently far in the past to work around - # filesystem-specific timestamp granularity. - os.utime(str(p), (old_mtime - 10, old_mtime - 10)) - # The file mtime should be refreshed by calling touch() again. - p.touch() - st = p.stat() - self.assertGreaterEqual(st.st_mtime_ns, old_mtime_ns) - self.assertGreaterEqual(st.st_mtime, old_mtime) - # Now with exist_ok=False. - p = P / 'newfileB' - self.assertFalse(p.exists()) - p.touch(mode=0o700, exist_ok=False) - self.assertTrue(p.exists()) - self.assertRaises(OSError, p.touch, exist_ok=False) - - def test_touch_nochange(self): - P = self.cls(BASE) - p = P / 'fileA' - p.touch() - with p.open('rb') as f: - self.assertEqual(f.read().strip(), b"this is file A") - - def test_mkdir(self): - P = self.cls(BASE) - p = P / 'newdirA' - self.assertFalse(p.exists()) - p.mkdir() - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - with self.assertRaises(OSError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - - def test_mkdir_parents(self): - # Creating a chain of directories. - p = self.cls(BASE, 'newdirB', 'newdirC') - self.assertFalse(p.exists()) - with self.assertRaises(OSError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.ENOENT) - p.mkdir(parents=True) - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - with self.assertRaises(OSError) as cm: - p.mkdir(parents=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - # Test `mode` arg. - mode = stat.S_IMODE(p.stat().st_mode) # Default mode. - p = self.cls(BASE, 'newdirD', 'newdirE') - p.mkdir(0o555, parents=True) - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - if os.name != 'nt': - # The directory's permissions follow the mode argument. - self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o7555 & mode) - # The parent's permissions follow the default process settings. - self.assertEqual(stat.S_IMODE(p.parent.stat().st_mode), mode) - - def test_mkdir_exist_ok(self): - p = self.cls(BASE, 'dirB') - st_ctime_first = p.stat().st_ctime - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - with self.assertRaises(FileExistsError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - p.mkdir(exist_ok=True) - self.assertTrue(p.exists()) - self.assertEqual(p.stat().st_ctime, st_ctime_first) - - def test_mkdir_exist_ok_with_parent(self): - p = self.cls(BASE, 'dirC') - self.assertTrue(p.exists()) - with self.assertRaises(FileExistsError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - p = p / 'newdirC' - p.mkdir(parents=True) - st_ctime_first = p.stat().st_ctime - self.assertTrue(p.exists()) - with self.assertRaises(FileExistsError) as cm: - p.mkdir(parents=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - p.mkdir(parents=True, exist_ok=True) - self.assertTrue(p.exists()) - self.assertEqual(p.stat().st_ctime, st_ctime_first) - - @unittest.skipIf(is_emscripten, "FS root cannot be modified on Emscripten.") - def test_mkdir_exist_ok_root(self): - # Issue #25803: A drive root could raise PermissionError on Windows. - self.cls('/').resolve().mkdir(exist_ok=True) - self.cls('/').resolve().mkdir(parents=True, exist_ok=True) - - @only_nt # XXX: not sure how to test this on POSIX. - def test_mkdir_with_unknown_drive(self): - for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': - p = self.cls(d + ':\\') - if not p.is_dir(): - break - else: - self.skipTest("cannot find a drive that doesn't exist") - with self.assertRaises(OSError): - (p / 'child' / 'path').mkdir(parents=True) - - def test_mkdir_with_child_file(self): - p = self.cls(BASE, 'dirB', 'fileB') - self.assertTrue(p.exists()) - # An exception is raised when the last path component is an existing - # regular file, regardless of whether exist_ok is true or not. - with self.assertRaises(FileExistsError) as cm: - p.mkdir(parents=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - with self.assertRaises(FileExistsError) as cm: - p.mkdir(parents=True, exist_ok=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - - def test_mkdir_no_parents_file(self): - p = self.cls(BASE, 'fileA') - self.assertTrue(p.exists()) - # An exception is raised when the last path component is an existing - # regular file, regardless of whether exist_ok is true or not. - with self.assertRaises(FileExistsError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - with self.assertRaises(FileExistsError) as cm: - p.mkdir(exist_ok=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - - def test_mkdir_concurrent_parent_creation(self): - for pattern_num in range(32): - p = self.cls(BASE, 'dirCPC%d' % pattern_num) - self.assertFalse(p.exists()) - - real_mkdir = os.mkdir - def my_mkdir(path, mode=0o777): - path = str(path) - # Emulate another process that would create the directory - # just before we try to create it ourselves. We do it - # in all possible pattern combinations, assuming that this - # function is called at most 5 times (dirCPC/dir1/dir2, - # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). - if pattern.pop(): - real_mkdir(path, mode) # From another process. - concurrently_created.add(path) - real_mkdir(path, mode) # Our real call. - - pattern = [bool(pattern_num & (1 << n)) for n in range(5)] - concurrently_created = set() - p12 = p / 'dir1' / 'dir2' - try: - with mock.patch("os.mkdir", my_mkdir): - p12.mkdir(parents=True, exist_ok=False) - except FileExistsError: - self.assertIn(str(p12), concurrently_created) - else: - self.assertNotIn(str(p12), concurrently_created) - self.assertTrue(p.exists()) - - def test_symlink_to(self): - if not self.can_symlink: - self.skipTest("symlinks required") - P = self.cls(BASE) - target = P / 'fileA' - # Symlinking a path target. - link = P / 'dirA' / 'linkAA' - link.symlink_to(target) - self.assertEqual(link.stat(), target.stat()) - self.assertNotEqual(link.lstat(), target.stat()) - # Symlinking a str target. - link = P / 'dirA' / 'linkAAA' - link.symlink_to(str(target)) - self.assertEqual(link.stat(), target.stat()) - self.assertNotEqual(link.lstat(), target.stat()) - self.assertFalse(link.is_dir()) - # Symlinking to a directory. - target = P / 'dirB' - link = P / 'dirA' / 'linkAAAA' - link.symlink_to(target, target_is_directory=True) - self.assertEqual(link.stat(), target.stat()) - self.assertNotEqual(link.lstat(), target.stat()) - self.assertTrue(link.is_dir()) - self.assertTrue(list(link.iterdir())) - - @unittest.skipIf(hasattr(os, "symlink"), "os.symlink() is present") - def test_symlink_to_unsupported(self): - P = self.cls(BASE) - p = P / 'fileA' - # linking to another path. - q = P / 'dirA' / 'fileAA' - with self.assertRaises(pathlib.UnsupportedOperation): - q.symlink_to(p) - - def test_is_junction(self): - P = self.cls(BASE) - - with mock.patch.object(P.pathmod, 'isjunction'): - self.assertEqual(P.is_junction(), P.pathmod.isjunction.return_value) - P.pathmod.isjunction.assert_called_once_with(P) - - @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") - @unittest.skipIf(sys.platform == "vxworks", - "fifo requires special path on VxWorks") - def test_is_fifo_true(self): - P = self.cls(BASE, 'myfifo') - try: - os.mkfifo(str(P)) - except PermissionError as e: - self.skipTest('os.mkfifo(): %s' % e) - self.assertTrue(P.is_fifo()) - self.assertFalse(P.is_socket()) - self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'myfifo\udfff').is_fifo(), False) - self.assertIs(self.cls(BASE, 'myfifo\x00').is_fifo(), False) - - @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") - @unittest.skipIf( - is_emscripten, "Unix sockets are not implemented on Emscripten." - ) - @unittest.skipIf( - is_wasi, "Cannot create socket on WASI." - ) - def test_is_socket_true(self): - P = self.cls(BASE, 'mysock') - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.addCleanup(sock.close) - try: - sock.bind(str(P)) - except OSError as e: - if (isinstance(e, PermissionError) or - "AF_UNIX path too long" in str(e)): - self.skipTest("cannot bind Unix socket: " + str(e)) - self.assertTrue(P.is_socket()) - self.assertFalse(P.is_fifo()) - self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'mysock\udfff').is_socket(), False) - self.assertIs(self.cls(BASE, 'mysock\x00').is_socket(), False) - - def test_is_char_device_true(self): - # Under Unix, /dev/null should generally be a char device. - P = self.cls('/dev/null') - if not P.exists(): - self.skipTest("/dev/null required") - self.assertTrue(P.is_char_device()) - self.assertFalse(P.is_block_device()) - self.assertFalse(P.is_file()) - self.assertIs(self.cls('/dev/null\udfff').is_char_device(), False) - self.assertIs(self.cls('/dev/null\x00').is_char_device(), False) - - def test_is_mount_root(self): - if os.name == 'nt': - R = self.cls('c:\\') - else: - R = self.cls('/') - self.assertTrue(R.is_mount()) - self.assertFalse((R / '\udfff').is_mount()) - - def test_passing_kwargs_deprecated(self): - with self.assertWarns(DeprecationWarning): - self.cls(foo="bar") - - def setUpWalk(self): - super().setUpWalk() - sub21_path= self.sub2_path / "SUB21" - tmp5_path = sub21_path / "tmp3" - broken_link3_path = self.sub2_path / "broken_link3" - - os.makedirs(sub21_path) - tmp5_path.write_text("I am tmp5, blame test_pathlib.") - if self.can_symlink: - os.symlink(tmp5_path, broken_link3_path) - self.sub2_tree[2].append('broken_link3') - self.sub2_tree[2].sort() - if not is_emscripten: - # Emscripten fails with inaccessible directories. - os.chmod(sub21_path, 0) - try: - os.listdir(sub21_path) - except PermissionError: - self.sub2_tree[1].append('SUB21') - else: - os.chmod(sub21_path, stat.S_IRWXU) - os.unlink(tmp5_path) - os.rmdir(sub21_path) - - def test_walk_bad_dir(self): - self.setUpWalk() - errors = [] - walk_it = self.walk_path.walk(on_error=errors.append) - root, dirs, files = next(walk_it) - self.assertEqual(errors, []) - dir1 = 'SUB1' - path1 = root / dir1 - path1new = (root / dir1).with_suffix(".new") - path1.rename(path1new) - try: - roots = [r for r, _, _ in walk_it] - self.assertTrue(errors) - self.assertNotIn(path1, roots) - self.assertNotIn(path1new, roots) - for dir2 in dirs: - if dir2 != dir1: - self.assertIn(root / dir2, roots) - finally: - path1new.rename(path1) - - def test_walk_many_open_files(self): - depth = 30 - base = self.cls(BASE, 'deep') - path = self.cls(base, *(['d']*depth)) - path.mkdir(parents=True) - - iters = [base.walk(top_down=False) for _ in range(100)] - for i in range(depth + 1): - expected = (path, ['d'] if i else [], []) - for it in iters: - self.assertEqual(next(it), expected) - path = path.parent - - iters = [base.walk(top_down=True) for _ in range(100)] - path = base - for i in range(depth + 1): - expected = (path, ['d'] if i < depth else [], []) - for it in iters: - self.assertEqual(next(it), expected) - path = path / 'd' - - -@only_posix -class PosixPathTest(PathTest, PurePosixPathTest): - cls = pathlib.PosixPath - - def test_absolute(self): - P = self.cls - self.assertEqual(str(P('/').absolute()), '/') - self.assertEqual(str(P('/a').absolute()), '/a') - self.assertEqual(str(P('/a/b').absolute()), '/a/b') - - # '//'-prefixed absolute path (supported by POSIX). - self.assertEqual(str(P('//').absolute()), '//') - self.assertEqual(str(P('//a').absolute()), '//a') - self.assertEqual(str(P('//a/b').absolute()), '//a/b') - - @unittest.skipIf( - is_emscripten or is_wasi, - "umask is not implemented on Emscripten/WASI." - ) - def test_open_mode(self): - old_mask = os.umask(0) - self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) - with (p / 'new_file').open('wb'): - pass - st = os.stat(join('new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) - with (p / 'other_new_file').open('wb'): - pass - st = os.stat(join('other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) - - def test_resolve_root(self): - current_directory = os.getcwd() - try: - os.chdir('/') - p = self.cls('spam') - self.assertEqual(str(p.resolve()), '/spam') - finally: - os.chdir(current_directory) - - @unittest.skipIf( - is_emscripten or is_wasi, - "umask is not implemented on Emscripten/WASI." - ) - def test_touch_mode(self): - old_mask = os.umask(0) - self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) - (p / 'new_file').touch() - st = os.stat(join('new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) - (p / 'other_new_file').touch() - st = os.stat(join('other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) - (p / 'masked_new_file').touch(mode=0o750) - st = os.stat(join('masked_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) - - def test_glob(self): - P = self.cls - p = P(BASE) - given = set(p.glob("FILEa")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.glob("FILEa*")), set()) - - def test_rglob(self): - P = self.cls - p = P(BASE, "dirC") - given = set(p.rglob("FILEd")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.rglob("FILEd*")), set()) - - @unittest.skipUnless(hasattr(pwd, 'getpwall'), - 'pwd module does not expose getpwall()') - @unittest.skipIf(sys.platform == "vxworks", - "no home directory on VxWorks") - def test_expanduser(self): - P = self.cls - import_helper.import_module('pwd') - import pwd - pwdent = pwd.getpwuid(os.getuid()) - username = pwdent.pw_name - userhome = pwdent.pw_dir.rstrip('/') or '/' - # Find arbitrary different user (if exists). - for pwdent in pwd.getpwall(): - othername = pwdent.pw_name - otherhome = pwdent.pw_dir.rstrip('/') - if othername != username and otherhome: - break - else: - othername = username - otherhome = userhome - - fakename = 'fakeuser' - # This user can theoretically exist on a test runner. Create unique name: - try: - while pwd.getpwnam(fakename): - fakename += '1' - except KeyError: - pass # Non-existent name found - - p1 = P('~/Documents') - p2 = P(f'~{username}/Documents') - p3 = P(f'~{othername}/Documents') - p4 = P(f'../~{username}/Documents') - p5 = P(f'/~{username}/Documents') - p6 = P('') - p7 = P(f'~{fakename}/Documents') - - with os_helper.EnvironmentVarGuard() as env: - env.pop('HOME', None) - - self.assertEqual(p1.expanduser(), P(userhome) / 'Documents') - self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') - self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - self.assertRaises(RuntimeError, p7.expanduser) - - env['HOME'] = '/tmp' - self.assertEqual(p1.expanduser(), P('/tmp/Documents')) - self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') - self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - self.assertRaises(RuntimeError, p7.expanduser) - - @unittest.skipIf(sys.platform != "darwin", - "Bad file descriptor in /dev/fd affects only macOS") - def test_handling_bad_descriptor(self): - try: - file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:] - if not file_descriptors: - self.skipTest("no file descriptors - issue was not reproduced") - # Checking all file descriptors because there is no guarantee - # which one will fail. - for f in file_descriptors: - f.exists() - f.is_dir() - f.is_file() - f.is_symlink() - f.is_block_device() - f.is_char_device() - f.is_fifo() - f.is_socket() - except OSError as e: - if e.errno == errno.EBADF: - self.fail("Bad file descriptor not handled.") - raise - - def test_from_uri(self): - P = self.cls - self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar')) - self.assertEqual(P.from_uri('file://foo/bar'), P('//foo/bar')) - self.assertEqual(P.from_uri('file:///foo/bar'), P('/foo/bar')) - self.assertEqual(P.from_uri('file:////foo/bar'), P('//foo/bar')) - self.assertEqual(P.from_uri('file://localhost/foo/bar'), P('/foo/bar')) - self.assertRaises(ValueError, P.from_uri, 'foo/bar') - self.assertRaises(ValueError, P.from_uri, '/foo/bar') - self.assertRaises(ValueError, P.from_uri, '//foo/bar') - self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') - self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') - - def test_from_uri_pathname2url(self): - P = self.cls - self.assertEqual(P.from_uri('file:' + pathname2url('/foo/bar')), P('/foo/bar')) - self.assertEqual(P.from_uri('file:' + pathname2url('//foo/bar')), P('//foo/bar')) - - -@only_nt -class WindowsPathTest(PathTest, PureWindowsPathTest): - cls = pathlib.WindowsPath - - def test_absolute(self): - P = self.cls - - # Simple absolute paths. - self.assertEqual(str(P('c:\\').absolute()), 'c:\\') - self.assertEqual(str(P('c:\\a').absolute()), 'c:\\a') - self.assertEqual(str(P('c:\\a\\b').absolute()), 'c:\\a\\b') - - # UNC absolute paths. - share = '\\\\server\\share\\' - self.assertEqual(str(P(share).absolute()), share) - self.assertEqual(str(P(share + 'a').absolute()), share + 'a') - self.assertEqual(str(P(share + 'a\\b').absolute()), share + 'a\\b') - - # UNC relative paths. - with mock.patch("os.getcwd") as getcwd: - getcwd.return_value = share - - self.assertEqual(str(P().absolute()), share) - self.assertEqual(str(P('.').absolute()), share) - self.assertEqual(str(P('a').absolute()), os.path.join(share, 'a')) - self.assertEqual(str(P('a', 'b', 'c').absolute()), - os.path.join(share, 'a', 'b', 'c')) - - drive = os.path.splitdrive(BASE)[0] - with os_helper.change_cwd(BASE): - # Relative path with root - self.assertEqual(str(P('\\').absolute()), drive + '\\') - self.assertEqual(str(P('\\foo').absolute()), drive + '\\foo') - - # Relative path on current drive - self.assertEqual(str(P(drive).absolute()), BASE) - self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(BASE, 'foo')) - - with os_helper.subst_drive(BASE) as other_drive: - # Set the working directory on the substitute drive - saved_cwd = os.getcwd() - other_cwd = f'{other_drive}\\dirA' - os.chdir(other_cwd) - os.chdir(saved_cwd) - - # Relative path on another drive - self.assertEqual(str(P(other_drive).absolute()), other_cwd) - self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') - - def test_glob(self): - P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("FILEa")), { P(BASE, "fileA") }) - self.assertEqual(set(p.glob("*a\\")), { P(BASE, "dirA") }) - self.assertEqual(set(p.glob("F*a")), { P(BASE, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) - self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) - - def test_rglob(self): - P = self.cls - p = P(BASE, "dirC") - self.assertEqual(set(p.rglob("FILEd")), { P(BASE, "dirC/dirD/fileD") }) - self.assertEqual(set(p.rglob("*\\")), { P(BASE, "dirC/dirD") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) - - def test_expanduser(self): - P = self.cls - with os_helper.EnvironmentVarGuard() as env: - env.pop('HOME', None) - env.pop('USERPROFILE', None) - env.pop('HOMEPATH', None) - env.pop('HOMEDRIVE', None) - env['USERNAME'] = 'alice' - - # test that the path returns unchanged - p1 = P('~/My Documents') - p2 = P('~alice/My Documents') - p3 = P('~bob/My Documents') - p4 = P('/~/My Documents') - p5 = P('d:~/My Documents') - p6 = P('') - self.assertRaises(RuntimeError, p1.expanduser) - self.assertRaises(RuntimeError, p2.expanduser) - self.assertRaises(RuntimeError, p3.expanduser) - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - - def check(): - env.pop('USERNAME', None) - self.assertEqual(p1.expanduser(), - P('C:/Users/alice/My Documents')) - self.assertRaises(RuntimeError, p2.expanduser) - env['USERNAME'] = 'alice' - self.assertEqual(p2.expanduser(), - P('C:/Users/alice/My Documents')) - self.assertEqual(p3.expanduser(), - P('C:/Users/bob/My Documents')) - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - - env['HOMEPATH'] = 'C:\\Users\\alice' - check() - - env['HOMEDRIVE'] = 'C:\\' - env['HOMEPATH'] = 'Users\\alice' - check() - - env.pop('HOMEDRIVE', None) - env.pop('HOMEPATH', None) - env['USERPROFILE'] = 'C:\\Users\\alice' - check() - - # bpo-38883: ignore `HOME` when set on windows - env['HOME'] = 'C:\\Users\\eve' - check() - - def test_from_uri(self): - P = self.cls - # DOS drive paths - self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file:c|/path/to/file'), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file:/c|/path/to/file'), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file:///c|/path/to/file'), P('c:/path/to/file')) - # UNC paths - self.assertEqual(P.from_uri('file://server/path/to/file'), P('//server/path/to/file')) - self.assertEqual(P.from_uri('file:////server/path/to/file'), P('//server/path/to/file')) - self.assertEqual(P.from_uri('file://///server/path/to/file'), P('//server/path/to/file')) - # Localhost paths - self.assertEqual(P.from_uri('file://localhost/c:/path/to/file'), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file://localhost/c|/path/to/file'), P('c:/path/to/file')) - # Invalid paths - self.assertRaises(ValueError, P.from_uri, 'foo/bar') - self.assertRaises(ValueError, P.from_uri, 'c:/foo/bar') - self.assertRaises(ValueError, P.from_uri, '//foo/bar') - self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') - self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') - - def test_from_uri_pathname2url(self): - P = self.cls - self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file')) - self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file')) - - def test_owner(self): - P = self.cls - with self.assertRaises(pathlib.UnsupportedOperation): - P('c:/').owner() - - def test_group(self): - P = self.cls - with self.assertRaises(pathlib.UnsupportedOperation): - P('c:/').group() - - -class PathSubclassTest(PathTest): - class cls(pathlib.Path): - pass - - # repr() roundtripping is not supported in custom subclass. - test_repr_roundtrips = None - - -class CompatiblePathTest(unittest.TestCase): - """ - Test that a type can be made compatible with PurePath - derivatives by implementing division operator overloads. - """ - - class CompatPath: - """ - Minimum viable class to test PurePath compatibility. - Simply uses the division operator to join a given - string and the string value of another object with - a forward slash. - """ - def __init__(self, string): - self.string = string - - def __truediv__(self, other): - return type(self)(f"{self.string}/{other}") - - def __rtruediv__(self, other): - return type(self)(f"{other}/{self.string}") - - def test_truediv(self): - result = pathlib.PurePath("test") / self.CompatPath("right") - self.assertIsInstance(result, self.CompatPath) - self.assertEqual(result.string, "test/right") - - with self.assertRaises(TypeError): - # Verify improper operations still raise a TypeError - pathlib.PurePath("test") / 10 - - def test_rtruediv(self): - result = self.CompatPath("left") / pathlib.PurePath("test") - self.assertIsInstance(result, self.CompatPath) - self.assertEqual(result.string, "left/test") - - with self.assertRaises(TypeError): - # Verify improper operations still raise a TypeError - 10 / pathlib.PurePath("test") - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_pathlib/__init__.py b/Lib/test/test_pathlib/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_pathlib/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py new file mode 100644 index 00000000000000..8f95c804f80e69 --- /dev/null +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -0,0 +1,2088 @@ +import io +import os +import sys +import errno +import ntpath +import pathlib +import pickle +import posixpath +import socket +import stat +import tempfile +import unittest +from unittest import mock +from urllib.request import pathname2url + +from test.support import import_helper +from test.support import is_emscripten, is_wasi +from test.support import set_recursion_limit +from test.support import os_helper +from test.support.os_helper import TESTFN, FakePath +from test.test_pathlib import test_pathlib_abc + +try: + import grp, pwd +except ImportError: + grp = pwd = None + + +only_nt = unittest.skipIf(os.name != 'nt', + 'test requires a Windows-compatible system') +only_posix = unittest.skipIf(os.name == 'nt', + 'test requires a POSIX-compatible system') + +root_in_posix = False +if hasattr(os, 'geteuid'): + root_in_posix = (os.geteuid() == 0) + +# +# Tests for the pure classes. +# + +class PurePathTest(test_pathlib_abc.DummyPurePathTest): + cls = pathlib.PurePath + + # Make sure any symbolic links in the base test path are resolved. + base = os.path.realpath(TESTFN) + + def test_concrete_class(self): + if self.cls is pathlib.PurePath: + expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_concrete_pathmod(self): + if self.cls is pathlib.PurePosixPath: + expected = posixpath + elif self.cls is pathlib.PureWindowsPath: + expected = ntpath + else: + expected = os.path + p = self.cls('a') + self.assertIs(p.pathmod, expected) + + def test_different_pathmods_unequal(self): + p = self.cls('a') + if p.pathmod is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + self.assertNotEqual(p, q) + + def test_different_pathmods_unordered(self): + p = self.cls('a') + if p.pathmod is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + with self.assertRaises(TypeError): + p < q + with self.assertRaises(TypeError): + p <= q + with self.assertRaises(TypeError): + p > q + with self.assertRaises(TypeError): + p >= q + + def test_constructor_nested(self): + P = self.cls + P(FakePath("a/b/c")) + self.assertEqual(P(P('a')), P('a')) + self.assertEqual(P(P('a'), 'b'), P('a/b')) + self.assertEqual(P(P('a'), P('b')), P('a/b')) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + self.assertEqual(P(P('./a:b')), P('./a:b')) + + def test_join_nested(self): + P = self.cls + p = P('a/b').joinpath(P('c')) + self.assertEqual(p, P('a/b/c')) + + def test_div_nested(self): + P = self.cls + p = P('a/b') / P('c') + self.assertEqual(p, P('a/b/c')) + + def test_pickling_common(self): + P = self.cls + for pathstr in ('a', 'a/', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c', 'a/b/c/'): + with self.subTest(pathstr=pathstr): + p = P(pathstr) + for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): + dumped = pickle.dumps(p, proto) + pp = pickle.loads(dumped) + self.assertIs(pp.__class__, p.__class__) + self.assertEqual(pp, p) + self.assertEqual(hash(pp), hash(p)) + self.assertEqual(str(pp), str(p)) + + def test_repr_common(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + clsname = p.__class__.__name__ + r = repr(p) + # The repr() is in the form ClassName("forward-slashes path"). + self.assertTrue(r.startswith(clsname + '('), r) + self.assertTrue(r.endswith(')'), r) + inner = r[len(clsname) + 1 : -1] + self.assertEqual(eval(inner), p.as_posix()) + + def test_fspath_common(self): + P = self.cls + p = P('a/b') + self._check_str(p.__fspath__(), ('a/b',)) + self._check_str(os.fspath(p), ('a/b',)) + + def test_bytes(self): + P = self.cls + message = (r"argument should be a str or an os\.PathLike object " + r"where __fspath__ returns a str, not 'bytes'") + with self.assertRaisesRegex(TypeError, message): + P(b'a') + with self.assertRaisesRegex(TypeError, message): + P(b'a', 'b') + with self.assertRaisesRegex(TypeError, message): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + + def test_as_bytes_common(self): + sep = os.fsencode(self.sep) + P = self.cls + self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') + + def test_ordering_common(self): + # Ordering is tuple-alike. + def assertLess(a, b): + self.assertLess(a, b) + self.assertGreater(b, a) + P = self.cls + a = P('a') + b = P('a/b') + c = P('abc') + d = P('b') + assertLess(a, b) + assertLess(a, c) + assertLess(a, d) + assertLess(b, c) + assertLess(c, d) + P = self.cls + a = P('/a') + b = P('/a/b') + c = P('/abc') + d = P('/b') + assertLess(a, b) + assertLess(a, c) + assertLess(a, d) + assertLess(b, c) + assertLess(c, d) + with self.assertRaises(TypeError): + P() < {} + + def test_as_uri_common(self): + P = self.cls + with self.assertRaises(ValueError): + P('a').as_uri() + with self.assertRaises(ValueError): + P().as_uri() + + def test_repr_roundtrips(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + r = repr(p) + # The repr() roundtrips. + q = eval(r, pathlib.__dict__) + self.assertIs(q.__class__, p.__class__) + self.assertEqual(q, p) + self.assertEqual(repr(q), r) + + +class PurePosixPathTest(PurePathTest): + cls = pathlib.PurePosixPath + + def test_parse_path(self): + check = self._check_parse_path + # Collapsing of excess leading slashes, except for the double-slash + # special case. + check('//a/b', '', '//', ['a', 'b']) + check('///a/b', '', '/', ['a', 'b']) + check('////a/b', '', '/', ['a', 'b']) + # Paths which look like NT paths aren't treated specially. + check('c:a', '', '', ['c:a',]) + check('c:\\a', '', '', ['c:\\a',]) + check('\\a', '', '', ['\\a',]) + + def test_root(self): + P = self.cls + self.assertEqual(P('/a/b').root, '/') + self.assertEqual(P('///a/b').root, '/') + # POSIX special case for two leading slashes. + self.assertEqual(P('//a/b').root, '//') + + def test_eq(self): + P = self.cls + self.assertNotEqual(P('a/b'), P('A/b')) + self.assertEqual(P('/a'), P('///a')) + self.assertNotEqual(P('/a'), P('//a')) + + def test_as_uri(self): + P = self.cls + self.assertEqual(P('/').as_uri(), 'file:///') + self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c') + self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c') + + def test_as_uri_non_ascii(self): + from urllib.parse import quote_from_bytes + P = self.cls + try: + os.fsencode('\xe9') + except UnicodeEncodeError: + self.skipTest("\\xe9 cannot be encoded to the filesystem encoding") + self.assertEqual(P('/a/b\xe9').as_uri(), + 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9'))) + + def test_match(self): + P = self.cls + self.assertFalse(P('A.py').match('a.PY')) + + def test_is_absolute(self): + P = self.cls + self.assertFalse(P().is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertTrue(P('/').is_absolute()) + self.assertTrue(P('/a').is_absolute()) + self.assertTrue(P('/a/b/').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + + def test_is_reserved(self): + P = self.cls + self.assertIs(False, P('').is_reserved()) + self.assertIs(False, P('/').is_reserved()) + self.assertIs(False, P('/foo/bar').is_reserved()) + self.assertIs(False, P('/dev/con/PRN/NUL').is_reserved()) + + def test_join(self): + P = self.cls + p = P('//a') + pp = p.joinpath('b') + self.assertEqual(pp, P('//a/b')) + pp = P('/a').joinpath('//c') + self.assertEqual(pp, P('//c')) + pp = P('//a').joinpath('/c') + self.assertEqual(pp, P('/c')) + + def test_div(self): + # Basically the same as joinpath(). + P = self.cls + p = P('//a') + pp = p / 'b' + self.assertEqual(pp, P('//a/b')) + pp = P('/a') / '//c' + self.assertEqual(pp, P('//c')) + pp = P('//a') / '/c' + self.assertEqual(pp, P('/c')) + + def test_parse_windows_path(self): + P = self.cls + p = P('c:', 'a', 'b') + pp = P(pathlib.PureWindowsPath('c:\\a\\b')) + self.assertEqual(p, pp) + + +class PureWindowsPathTest(PurePathTest): + cls = pathlib.PureWindowsPath + + equivalences = PurePathTest.equivalences.copy() + equivalences.update({ + './a:b': [ ('./a:b',) ], + 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], + 'c:/a': [ + ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), + ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), + ], + '//a/b/': [ ('//a/b',) ], + '//a/b/c': [ + ('//a/b', 'c'), ('//a/b/', 'c'), + ], + }) + + def test_parse_path(self): + check = self._check_parse_path + # First part is anchored. + check('c:', 'c:', '', []) + check('c:/', 'c:', '\\', []) + check('/', '', '\\', []) + check('c:a', 'c:', '', ['a']) + check('c:/a', 'c:', '\\', ['a']) + check('/a', '', '\\', ['a']) + # UNC paths. + check('//', '\\\\', '', []) + check('//a', '\\\\a', '', []) + check('//a/', '\\\\a\\', '', []) + check('//a/b', '\\\\a\\b', '\\', []) + check('//a/b/', '\\\\a\\b', '\\', []) + check('//a/b/c', '\\\\a\\b', '\\', ['c']) + # Collapsing and stripping excess slashes. + check('Z://b//c/d/', 'Z:', '\\', ['b', 'c', 'd']) + # UNC paths. + check('//b/c//d', '\\\\b\\c', '\\', ['d']) + # Extended paths. + check('//./c:', '\\\\.\\c:', '', []) + check('//?/c:/', '\\\\?\\c:', '\\', []) + check('//?/c:/a', '\\\\?\\c:', '\\', ['a']) + # Extended UNC paths (format is "\\?\UNC\server\share"). + check('//?', '\\\\?', '', []) + check('//?/', '\\\\?\\', '', []) + check('//?/UNC', '\\\\?\\UNC', '', []) + check('//?/UNC/', '\\\\?\\UNC\\', '', []) + check('//?/UNC/b', '\\\\?\\UNC\\b', '', []) + check('//?/UNC/b/', '\\\\?\\UNC\\b\\', '', []) + check('//?/UNC/b/c', '\\\\?\\UNC\\b\\c', '\\', []) + check('//?/UNC/b/c/', '\\\\?\\UNC\\b\\c', '\\', []) + check('//?/UNC/b/c/d', '\\\\?\\UNC\\b\\c', '\\', ['d']) + # UNC device paths + check('//./BootPartition/', '\\\\.\\BootPartition', '\\', []) + check('//?/BootPartition/', '\\\\?\\BootPartition', '\\', []) + check('//./PhysicalDrive0', '\\\\.\\PhysicalDrive0', '', []) + check('//?/Volume{}/', '\\\\?\\Volume{}', '\\', []) + check('//./nul', '\\\\.\\nul', '', []) + # Paths to files with NTFS alternate data streams + check('./c:s', '', '', ['c:s']) + check('cc:s', '', '', ['cc:s']) + check('C:c:s', 'C:', '', ['c:s']) + check('C:/c:s', 'C:', '\\', ['c:s']) + check('D:a/c:b', 'D:', '', ['a', 'c:b']) + check('D:/a/c:b', 'D:', '\\', ['a', 'c:b']) + + def test_str(self): + p = self.cls('a/b/c') + self.assertEqual(str(p), 'a\\b\\c') + p = self.cls('c:/a/b/c') + self.assertEqual(str(p), 'c:\\a\\b\\c') + p = self.cls('//a/b') + self.assertEqual(str(p), '\\\\a\\b\\') + p = self.cls('//a/b/c') + self.assertEqual(str(p), '\\\\a\\b\\c') + p = self.cls('//a/b/c/d') + self.assertEqual(str(p), '\\\\a\\b\\c\\d') + + def test_str_subclass(self): + self._check_str_subclass('.\\a:b') + self._check_str_subclass('c:') + self._check_str_subclass('c:a') + self._check_str_subclass('c:a\\b.txt') + self._check_str_subclass('c:\\') + self._check_str_subclass('c:\\a') + self._check_str_subclass('c:\\a\\b.txt') + self._check_str_subclass('\\\\some\\share') + self._check_str_subclass('\\\\some\\share\\a') + self._check_str_subclass('\\\\some\\share\\a\\b.txt') + + def test_eq(self): + P = self.cls + self.assertEqual(P('c:a/b'), P('c:a/b')) + self.assertEqual(P('c:a/b'), P('c:', 'a', 'b')) + self.assertNotEqual(P('c:a/b'), P('d:a/b')) + self.assertNotEqual(P('c:a/b'), P('c:/a/b')) + self.assertNotEqual(P('/a/b'), P('c:/a/b')) + # Case-insensitivity. + self.assertEqual(P('a/B'), P('A/b')) + self.assertEqual(P('C:a/B'), P('c:A/b')) + self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b')) + self.assertEqual(P('\u0130'), P('i\u0307')) + + def test_as_uri(self): + P = self.cls + with self.assertRaises(ValueError): + P('/a/b').as_uri() + with self.assertRaises(ValueError): + P('c:a/b').as_uri() + self.assertEqual(P('c:/').as_uri(), 'file:///c:/') + self.assertEqual(P('c:/a/b.c').as_uri(), 'file:///c:/a/b.c') + self.assertEqual(P('c:/a/b%#c').as_uri(), 'file:///c:/a/b%25%23c') + self.assertEqual(P('c:/a/b\xe9').as_uri(), 'file:///c:/a/b%C3%A9') + self.assertEqual(P('//some/share/').as_uri(), 'file://some/share/') + self.assertEqual(P('//some/share/a/b.c').as_uri(), + 'file://some/share/a/b.c') + self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(), + 'file://some/share/a/b%25%23c%C3%A9') + + def test_match(self): + P = self.cls + # Absolute patterns. + self.assertTrue(P('c:/b.py').match('*:/*.py')) + self.assertTrue(P('c:/b.py').match('c:/*.py')) + self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('b.py').match('c:*.py')) + self.assertFalse(P('b.py').match('c:/*.py')) + self.assertFalse(P('c:b.py').match('/*.py')) + self.assertFalse(P('c:b.py').match('c:/*.py')) + self.assertFalse(P('/b.py').match('c:*.py')) + self.assertFalse(P('/b.py').match('c:/*.py')) + # UNC patterns. + self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) + self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) + # Case-insensitivity. + self.assertTrue(P('B.py').match('b.PY')) + self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) + self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) + # Path anchor doesn't match pattern anchor + self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' + self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' + self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' + + def test_ordering_common(self): + # Case-insensitivity. + def assertOrderedEqual(a, b): + self.assertLessEqual(a, b) + self.assertGreaterEqual(b, a) + P = self.cls + p = P('c:A/b') + q = P('C:a/B') + assertOrderedEqual(p, q) + self.assertFalse(p < q) + self.assertFalse(p > q) + p = P('//some/Share/A/b') + q = P('//Some/SHARE/a/B') + assertOrderedEqual(p, q) + self.assertFalse(p < q) + self.assertFalse(p > q) + + def test_parts(self): + P = self.cls + p = P('c:a/b') + parts = p.parts + self.assertEqual(parts, ('c:', 'a', 'b')) + p = P('c:/a/b') + parts = p.parts + self.assertEqual(parts, ('c:\\', 'a', 'b')) + p = P('//a/b/c/d') + parts = p.parts + self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) + + def test_parent(self): + # Anchored + P = self.cls + p = P('z:a/b/c') + self.assertEqual(p.parent, P('z:a/b')) + self.assertEqual(p.parent.parent, P('z:a')) + self.assertEqual(p.parent.parent.parent, P('z:')) + self.assertEqual(p.parent.parent.parent.parent, P('z:')) + p = P('z:/a/b/c') + self.assertEqual(p.parent, P('z:/a/b')) + self.assertEqual(p.parent.parent, P('z:/a')) + self.assertEqual(p.parent.parent.parent, P('z:/')) + self.assertEqual(p.parent.parent.parent.parent, P('z:/')) + p = P('//a/b/c/d') + self.assertEqual(p.parent, P('//a/b/c')) + self.assertEqual(p.parent.parent, P('//a/b')) + self.assertEqual(p.parent.parent.parent, P('//a/b')) + + def test_parents(self): + # Anchored + P = self.cls + p = P('z:a/b/') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:a')) + self.assertEqual(par[1], P('z:')) + self.assertEqual(par[0:1], (P('z:a'),)) + self.assertEqual(par[:-1], (P('z:a'),)) + self.assertEqual(par[:2], (P('z:a'), P('z:'))) + self.assertEqual(par[1:], (P('z:'),)) + self.assertEqual(par[::2], (P('z:a'),)) + self.assertEqual(par[::-1], (P('z:'), P('z:a'))) + self.assertEqual(list(par), [P('z:a'), P('z:')]) + with self.assertRaises(IndexError): + par[2] + p = P('z:/a/b/') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:/a')) + self.assertEqual(par[1], P('z:/')) + self.assertEqual(par[0:1], (P('z:/a'),)) + self.assertEqual(par[0:-1], (P('z:/a'),)) + self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) + self.assertEqual(par[1:], (P('z:/'),)) + self.assertEqual(par[::2], (P('z:/a'),)) + self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) + self.assertEqual(list(par), [P('z:/a'), P('z:/')]) + with self.assertRaises(IndexError): + par[2] + p = P('//a/b/c/d') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('//a/b/c')) + self.assertEqual(par[1], P('//a/b')) + self.assertEqual(par[0:1], (P('//a/b/c'),)) + self.assertEqual(par[0:-1], (P('//a/b/c'),)) + self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) + self.assertEqual(par[1:], (P('//a/b'),)) + self.assertEqual(par[::2], (P('//a/b/c'),)) + self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) + self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) + with self.assertRaises(IndexError): + par[2] + + def test_drive(self): + P = self.cls + self.assertEqual(P('c:').drive, 'c:') + self.assertEqual(P('c:a/b').drive, 'c:') + self.assertEqual(P('c:/').drive, 'c:') + self.assertEqual(P('c:/a/b/').drive, 'c:') + self.assertEqual(P('//a/b').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') + + def test_root(self): + P = self.cls + self.assertEqual(P('c:').root, '') + self.assertEqual(P('c:a/b').root, '') + self.assertEqual(P('c:/').root, '\\') + self.assertEqual(P('c:/a/b/').root, '\\') + self.assertEqual(P('//a/b').root, '\\') + self.assertEqual(P('//a/b/').root, '\\') + self.assertEqual(P('//a/b/c/d').root, '\\') + + def test_anchor(self): + P = self.cls + self.assertEqual(P('c:').anchor, 'c:') + self.assertEqual(P('c:a/b').anchor, 'c:') + self.assertEqual(P('c:/').anchor, 'c:\\') + self.assertEqual(P('c:/a/b/').anchor, 'c:\\') + self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') + self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') + self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') + + def test_name(self): + P = self.cls + self.assertEqual(P('c:').name, '') + self.assertEqual(P('c:/').name, '') + self.assertEqual(P('c:a/b').name, 'b') + self.assertEqual(P('c:/a/b').name, 'b') + self.assertEqual(P('c:a/b.py').name, 'b.py') + self.assertEqual(P('c:/a/b.py').name, 'b.py') + self.assertEqual(P('//My.py/Share.php').name, '') + self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') + + def test_suffix(self): + P = self.cls + self.assertEqual(P('c:').suffix, '') + self.assertEqual(P('c:/').suffix, '') + self.assertEqual(P('c:a/b').suffix, '') + self.assertEqual(P('c:/a/b').suffix, '') + self.assertEqual(P('c:a/b.py').suffix, '.py') + self.assertEqual(P('c:/a/b.py').suffix, '.py') + self.assertEqual(P('c:a/.hgrc').suffix, '') + self.assertEqual(P('c:/a/.hgrc').suffix, '') + self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('//My.py/Share.php').suffix, '') + self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') + + def test_suffixes(self): + P = self.cls + self.assertEqual(P('c:').suffixes, []) + self.assertEqual(P('c:/').suffixes, []) + self.assertEqual(P('c:a/b').suffixes, []) + self.assertEqual(P('c:/a/b').suffixes, []) + self.assertEqual(P('c:a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:a/.hgrc').suffixes, []) + self.assertEqual(P('c:/a/.hgrc').suffixes, []) + self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('//My.py/Share.php').suffixes, []) + self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) + self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) + + def test_stem(self): + P = self.cls + self.assertEqual(P('c:').stem, '') + self.assertEqual(P('c:.').stem, '') + self.assertEqual(P('c:..').stem, '..') + self.assertEqual(P('c:/').stem, '') + self.assertEqual(P('c:a/b').stem, 'b') + self.assertEqual(P('c:a/b.py').stem, 'b') + self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') + self.assertEqual(P('c:a/.hg.rc').stem, '.hg') + self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, + 'Some name. Ending with a dot.') + + def test_with_name(self): + P = self.cls + self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml')) + self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml')) + self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml')) + self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml')) + self.assertRaises(ValueError, P('c:').with_name, 'd.xml') + self.assertRaises(ValueError, P('c:/').with_name, 'd.xml') + self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml') + self.assertEqual(str(P('a').with_name('d:')), '.\\d:') + self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e') + self.assertEqual(P('c:a/b').with_name('d:'), P('c:a/d:')) + self.assertEqual(P('c:a/b').with_name('d:e'), P('c:a/d:e')) + self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e') + self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share') + + def test_with_stem(self): + P = self.cls + self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) + self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) + self.assertRaises(ValueError, P('c:').with_stem, 'd') + self.assertRaises(ValueError, P('c:/').with_stem, 'd') + self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') + self.assertEqual(str(P('a').with_stem('d:')), '.\\d:') + self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e') + self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:')) + self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e')) + self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') + self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') + + def test_with_suffix(self): + P = self.cls + self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) + self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('.').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') + # Invalid suffix. + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') + + def test_relative_to(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) + self.assertEqual(p.relative_to('c:foO'), P('Bar')) + self.assertEqual(p.relative_to('c:foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR')), P()) + self.assertEqual(p.relative_to('c:foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) + self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) + self.assertRaises(ValueError, p.relative_to, '', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) + p = P('C:/Foo/Bar') + self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) + self.assertEqual(p.relative_to('c:/foO'), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) + self.assertEqual(p.relative_to('c:/foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) + self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, 'c:') + self.assertRaises(ValueError, p.relative_to, P('c:')) + self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo')) + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('d:/')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) + self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) + + def test_is_relative_to(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:'))) + self.assertTrue(p.is_relative_to('c:')) + self.assertTrue(p.is_relative_to(P('c:foO'))) + self.assertTrue(p.is_relative_to('c:foO')) + self.assertTrue(p.is_relative_to('c:foO/')) + self.assertTrue(p.is_relative_to(P('c:foO/baR'))) + self.assertTrue(p.is_relative_to('c:foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('Foo'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('C:/Foo'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) + p = P('C:/Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:/'))) + self.assertTrue(p.is_relative_to(P('c:/foO'))) + self.assertTrue(p.is_relative_to('c:/foO/')) + self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) + self.assertTrue(p.is_relative_to('c:/foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to('c:')) + self.assertFalse(p.is_relative_to(P('C:/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo'))) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('d:/'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('//C/Foo'))) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) + self.assertTrue(p.is_relative_to('//sErver/sHare')) + self.assertTrue(p.is_relative_to('//sErver/sHare/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) + + def test_is_absolute(self): + P = self.cls + # Under NT, only paths with both a drive and a root are absolute. + self.assertFalse(P().is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertFalse(P('/').is_absolute()) + self.assertFalse(P('/a').is_absolute()) + self.assertFalse(P('/a/b/').is_absolute()) + self.assertFalse(P('c:').is_absolute()) + self.assertFalse(P('c:a').is_absolute()) + self.assertFalse(P('c:a/b/').is_absolute()) + self.assertTrue(P('c:/').is_absolute()) + self.assertTrue(P('c:/a').is_absolute()) + self.assertTrue(P('c:/a/b/').is_absolute()) + # UNC paths are absolute by definition. + self.assertTrue(P('//a/b').is_absolute()) + self.assertTrue(P('//a/b/').is_absolute()) + self.assertTrue(P('//a/b/c').is_absolute()) + self.assertTrue(P('//a/b/c/d').is_absolute()) + + def test_join(self): + P = self.cls + p = P('C:/a/b') + pp = p.joinpath('x/y') + self.assertEqual(pp, P('C:/a/b/x/y')) + pp = p.joinpath('/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + pp = p.joinpath('D:x/y') + self.assertEqual(pp, P('D:x/y')) + pp = p.joinpath('D:/x/y') + self.assertEqual(pp, P('D:/x/y')) + pp = p.joinpath('//host/share/x/y') + self.assertEqual(pp, P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + pp = p.joinpath('c:x/y') + self.assertEqual(pp, P('C:/a/b/x/y')) + pp = p.joinpath('c:/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath(P('./d:s')) + self.assertEqual(pp, P('C:/a/b/d:s')) + pp = p.joinpath(P('./dd:s')) + self.assertEqual(pp, P('C:/a/b/dd:s')) + pp = p.joinpath(P('E:d:s')) + self.assertEqual(pp, P('E:d:s')) + # Joining onto a UNC path with no root + pp = P('//').joinpath('server') + self.assertEqual(pp, P('//server')) + pp = P('//server').joinpath('share') + self.assertEqual(pp, P('//server/share')) + pp = P('//./BootPartition').joinpath('Windows') + self.assertEqual(pp, P('//./BootPartition/Windows')) + + def test_div(self): + # Basically the same as joinpath(). + P = self.cls + p = P('C:/a/b') + self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) + self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) + self.assertEqual(p / '/x/y', P('C:/x/y')) + self.assertEqual(p / '/x' / 'y', P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + self.assertEqual(p / 'D:x/y', P('D:x/y')) + self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) + self.assertEqual(p / 'D:/x/y', P('D:/x/y')) + self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) + self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) + self.assertEqual(p / 'c:/x/y', P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) + self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) + self.assertEqual(p / P('E:d:s'), P('E:d:s')) + + def test_is_reserved(self): + P = self.cls + self.assertIs(False, P('').is_reserved()) + self.assertIs(False, P('/').is_reserved()) + self.assertIs(False, P('/foo/bar').is_reserved()) + # UNC paths are never reserved. + self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) + # Case-insensitive DOS-device names are reserved. + self.assertIs(True, P('nul').is_reserved()) + self.assertIs(True, P('aux').is_reserved()) + self.assertIs(True, P('prn').is_reserved()) + self.assertIs(True, P('con').is_reserved()) + self.assertIs(True, P('conin$').is_reserved()) + self.assertIs(True, P('conout$').is_reserved()) + # COM/LPT + 1-9 or + superscript 1-3 are reserved. + self.assertIs(True, P('COM1').is_reserved()) + self.assertIs(True, P('LPT9').is_reserved()) + self.assertIs(True, P('com\xb9').is_reserved()) + self.assertIs(True, P('com\xb2').is_reserved()) + self.assertIs(True, P('lpt\xb3').is_reserved()) + # DOS-device name mataching ignores characters after a dot or + # a colon and also ignores trailing spaces. + self.assertIs(True, P('NUL.txt').is_reserved()) + self.assertIs(True, P('PRN ').is_reserved()) + self.assertIs(True, P('AUX .txt').is_reserved()) + self.assertIs(True, P('COM1:bar').is_reserved()) + self.assertIs(True, P('LPT9 :bar').is_reserved()) + # DOS-device names are only matched at the beginning + # of a path component. + self.assertIs(False, P('bar.com9').is_reserved()) + self.assertIs(False, P('bar.lpt9').is_reserved()) + # Only the last path component matters. + self.assertIs(True, P('c:/baz/con/NUL').is_reserved()) + self.assertIs(False, P('c:/NUL/con/baz').is_reserved()) + + +class PurePathSubclassTest(PurePathTest): + class cls(pathlib.PurePath): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + +# +# Tests for the concrete classes. +# + +class PathTest(test_pathlib_abc.DummyPathTest, PurePathTest): + """Tests for the FS-accessing functionalities of the Path classes.""" + cls = pathlib.Path + can_symlink = os_helper.can_symlink() + + def setUp(self): + super().setUp() + os.chmod(self.pathmod.join(self.base, 'dirE'), 0) + + def tearDown(self): + os.chmod(self.pathmod.join(self.base, 'dirE'), 0o777) + os_helper.rmtree(self.base) + + def tempdir(self): + d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', + dir=os.getcwd())) + self.addCleanup(os_helper.rmtree, d) + return d + + def test_matches_pathbase_api(self): + our_names = {name for name in dir(self.cls) if name[0] != '_'} + path_names = {name for name in dir(pathlib._abc.PathBase) if name[0] != '_'} + self.assertEqual(our_names, path_names) + for attr_name in our_names: + if attr_name == 'pathmod': + # On Windows, Path.pathmod is ntpath, but PathBase.pathmod is + # posixpath, and so their docstrings differ. + continue + our_attr = getattr(self.cls, attr_name) + path_attr = getattr(pathlib._abc.PathBase, attr_name) + self.assertEqual(our_attr.__doc__, path_attr.__doc__) + + def test_concrete_class(self): + if self.cls is pathlib.Path: + expected = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_unsupported_pathmod(self): + if self.cls.pathmod is os.path: + self.skipTest("path flavour is supported") + else: + self.assertRaises(pathlib.UnsupportedOperation, self.cls) + + def _test_cwd(self, p): + q = self.cls(os.getcwd()) + self.assertEqual(p, q) + self.assertEqualNormCase(str(p), str(q)) + self.assertIs(type(p), type(q)) + self.assertTrue(p.is_absolute()) + + def test_cwd(self): + p = self.cls.cwd() + self._test_cwd(p) + + def test_absolute_common(self): + P = self.cls + + with mock.patch("os.getcwd") as getcwd: + getcwd.return_value = self.base + + # Simple relative paths. + self.assertEqual(str(P().absolute()), self.base) + self.assertEqual(str(P('.').absolute()), self.base) + self.assertEqual(str(P('a').absolute()), os.path.join(self.base, 'a')) + self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(self.base, 'a', 'b', 'c')) + + # Symlinks should not be resolved. + self.assertEqual(str(P('linkB', 'fileB').absolute()), os.path.join(self.base, 'linkB', 'fileB')) + self.assertEqual(str(P('brokenLink').absolute()), os.path.join(self.base, 'brokenLink')) + self.assertEqual(str(P('brokenLinkLoop').absolute()), os.path.join(self.base, 'brokenLinkLoop')) + + # '..' entries should be preserved and not normalised. + self.assertEqual(str(P('..').absolute()), os.path.join(self.base, '..')) + self.assertEqual(str(P('a', '..').absolute()), os.path.join(self.base, 'a', '..')) + self.assertEqual(str(P('..', 'b').absolute()), os.path.join(self.base, '..', 'b')) + + def _test_home(self, p): + q = self.cls(os.path.expanduser('~')) + self.assertEqual(p, q) + self.assertEqualNormCase(str(p), str(q)) + self.assertIs(type(p), type(q)) + self.assertTrue(p.is_absolute()) + + @unittest.skipIf( + pwd is None, reason="Test requires pwd module to get homedir." + ) + def test_home(self): + with os_helper.EnvironmentVarGuard() as env: + self._test_home(self.cls.home()) + + env.clear() + env['USERPROFILE'] = os.path.join(self.base, 'userprofile') + self._test_home(self.cls.home()) + + # bpo-38883: ignore `HOME` when set on windows + env['HOME'] = os.path.join(self.base, 'home') + self._test_home(self.cls.home()) + + @unittest.skipIf(is_wasi, "WASI has no user accounts.") + def test_expanduser_common(self): + P = self.cls + p = P('~') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'))) + p = P('foo') + self.assertEqual(p.expanduser(), p) + p = P('/~') + self.assertEqual(p.expanduser(), p) + p = P('../~') + self.assertEqual(p.expanduser(), p) + p = P(P('').absolute().anchor) / '~' + self.assertEqual(p.expanduser(), p) + p = P('~/a:b') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) + + def test_with_segments(self): + class P(self.cls): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + p = P(self.base, session_id=42) + self.assertEqual(42, p.absolute().session_id) + self.assertEqual(42, p.resolve().session_id) + if not is_wasi: # WASI has no user accounts. + self.assertEqual(42, p.with_segments('~').expanduser().session_id) + self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) + self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) + if self.can_symlink: + self.assertEqual(42, (p / 'linkA').readlink().session_id) + for path in p.iterdir(): + self.assertEqual(42, path.session_id) + for path in p.glob('*'): + self.assertEqual(42, path.session_id) + for path in p.rglob('*'): + self.assertEqual(42, path.session_id) + for dirpath, dirnames, filenames in p.walk(): + self.assertEqual(42, dirpath.session_id) + + def test_open_unbuffered(self): + p = self.cls(self.base) + with (p / 'fileA').open('rb', buffering=0) as f: + self.assertIsInstance(f, io.RawIOBase) + self.assertEqual(f.read().strip(), b"this is file A") + + def test_resolve_nonexist_relative_issue38671(self): + p = self.cls('non', 'exist') + + old_cwd = os.getcwd() + os.chdir(self.base) + try: + self.assertEqual(p.resolve(), self.cls(self.base, p)) + finally: + os.chdir(old_cwd) + + @os_helper.skip_unless_working_chmod + def test_chmod(self): + p = self.cls(self.base) / 'fileA' + mode = p.stat().st_mode + # Clear writable bit. + new_mode = mode & ~0o222 + p.chmod(new_mode) + self.assertEqual(p.stat().st_mode, new_mode) + # Set writable bit. + new_mode = mode | 0o222 + p.chmod(new_mode) + self.assertEqual(p.stat().st_mode, new_mode) + + # On Windows, os.chmod does not follow symlinks (issue #15411) + @only_posix + @os_helper.skip_unless_working_chmod + def test_chmod_follow_symlinks_true(self): + p = self.cls(self.base) / 'linkA' + q = p.resolve() + mode = q.stat().st_mode + # Clear writable bit. + new_mode = mode & ~0o222 + p.chmod(new_mode, follow_symlinks=True) + self.assertEqual(q.stat().st_mode, new_mode) + # Set writable bit + new_mode = mode | 0o222 + p.chmod(new_mode, follow_symlinks=True) + self.assertEqual(q.stat().st_mode, new_mode) + + # XXX also need a test for lchmod. + + def _get_pw_name_or_skip_test(self, uid): + try: + return pwd.getpwuid(uid).pw_name + except KeyError: + self.skipTest( + "user %d doesn't have an entry in the system database" % uid) + + @unittest.skipUnless(pwd, "the pwd module is needed for this test") + def test_owner(self): + p = self.cls(self.base) / 'fileA' + expected_uid = p.stat().st_uid + expected_name = self._get_pw_name_or_skip_test(expected_uid) + + self.assertEqual(expected_name, p.owner()) + + @unittest.skipUnless(pwd, "the pwd module is needed for this test") + @unittest.skipUnless(root_in_posix, "test needs root privilege") + def test_owner_no_follow_symlinks(self): + all_users = [u.pw_uid for u in pwd.getpwall()] + if len(all_users) < 2: + self.skipTest("test needs more than one user") + + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' + + uid_1, uid_2 = all_users[:2] + os.chown(target, uid_1, -1) + os.chown(link, uid_2, -1, follow_symlinks=False) + + expected_uid = link.stat(follow_symlinks=False).st_uid + expected_name = self._get_pw_name_or_skip_test(expected_uid) + + self.assertEqual(expected_uid, uid_2) + self.assertEqual(expected_name, link.owner(follow_symlinks=False)) + + def _get_gr_name_or_skip_test(self, gid): + try: + return grp.getgrgid(gid).gr_name + except KeyError: + self.skipTest( + "group %d doesn't have an entry in the system database" % gid) + + @unittest.skipUnless(grp, "the grp module is needed for this test") + def test_group(self): + p = self.cls(self.base) / 'fileA' + expected_gid = p.stat().st_gid + expected_name = self._get_gr_name_or_skip_test(expected_gid) + + self.assertEqual(expected_name, p.group()) + + @unittest.skipUnless(grp, "the grp module is needed for this test") + @unittest.skipUnless(root_in_posix, "test needs root privilege") + def test_group_no_follow_symlinks(self): + all_groups = [g.gr_gid for g in grp.getgrall()] + if len(all_groups) < 2: + self.skipTest("test needs more than one group") + + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' + + gid_1, gid_2 = all_groups[:2] + os.chown(target, -1, gid_1) + os.chown(link, -1, gid_2, follow_symlinks=False) + + expected_gid = link.stat(follow_symlinks=False).st_gid + expected_name = self._get_pw_name_or_skip_test(expected_gid) + + self.assertEqual(expected_gid, gid_2) + self.assertEqual(expected_name, link.group(follow_symlinks=False)) + + def test_unlink(self): + p = self.cls(self.base) / 'fileA' + p.unlink() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_unlink_missing_ok(self): + p = self.cls(self.base) / 'fileAAA' + self.assertFileNotFound(p.unlink) + p.unlink(missing_ok=True) + + def test_rmdir(self): + p = self.cls(self.base) / 'dirA' + for q in p.iterdir(): + q.unlink() + p.rmdir() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") + def test_hardlink_to(self): + P = self.cls(self.base) + target = P / 'fileA' + size = target.stat().st_size + # linking to another path. + link = P / 'dirA' / 'fileAA' + link.hardlink_to(target) + self.assertEqual(link.stat().st_size, size) + self.assertTrue(os.path.samefile(target, link)) + self.assertTrue(target.exists()) + # Linking to a str of a relative path. + link2 = P / 'dirA' / 'fileAAA' + target2 = self.pathmod.join(TESTFN, 'fileA') + link2.hardlink_to(target2) + self.assertEqual(os.stat(target2).st_size, size) + self.assertTrue(link2.exists()) + + @unittest.skipIf(hasattr(os, "link"), "os.link() is present") + def test_hardlink_to_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + # linking to another path. + q = P / 'dirA' / 'fileAA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.hardlink_to(p) + + def test_rename(self): + P = self.cls(self.base) + p = P / 'fileA' + size = p.stat().st_size + # Renaming to another path. + q = P / 'dirA' / 'fileAA' + renamed_p = p.rename(q) + self.assertEqual(renamed_p, q) + self.assertEqual(q.stat().st_size, size) + self.assertFileNotFound(p.stat) + # Renaming to a str of a relative path. + r = self.pathmod.join(TESTFN, 'fileAAA') + renamed_q = q.rename(r) + self.assertEqual(renamed_q, self.cls(r)) + self.assertEqual(os.stat(r).st_size, size) + self.assertFileNotFound(q.stat) + + def test_replace(self): + P = self.cls(self.base) + p = P / 'fileA' + size = p.stat().st_size + # Replacing a non-existing path. + q = P / 'dirA' / 'fileAA' + replaced_p = p.replace(q) + self.assertEqual(replaced_p, q) + self.assertEqual(q.stat().st_size, size) + self.assertFileNotFound(p.stat) + # Replacing another (existing) path. + r = self.pathmod.join(TESTFN, 'dirB', 'fileB') + replaced_q = q.replace(r) + self.assertEqual(replaced_q, self.cls(r)) + self.assertEqual(os.stat(r).st_size, size) + self.assertFileNotFound(q.stat) + + def test_touch_common(self): + P = self.cls(self.base) + p = P / 'newfileA' + self.assertFalse(p.exists()) + p.touch() + self.assertTrue(p.exists()) + st = p.stat() + old_mtime = st.st_mtime + old_mtime_ns = st.st_mtime_ns + # Rewind the mtime sufficiently far in the past to work around + # filesystem-specific timestamp granularity. + os.utime(str(p), (old_mtime - 10, old_mtime - 10)) + # The file mtime should be refreshed by calling touch() again. + p.touch() + st = p.stat() + self.assertGreaterEqual(st.st_mtime_ns, old_mtime_ns) + self.assertGreaterEqual(st.st_mtime, old_mtime) + # Now with exist_ok=False. + p = P / 'newfileB' + self.assertFalse(p.exists()) + p.touch(mode=0o700, exist_ok=False) + self.assertTrue(p.exists()) + self.assertRaises(OSError, p.touch, exist_ok=False) + + def test_touch_nochange(self): + P = self.cls(self.base) + p = P / 'fileA' + p.touch() + with p.open('rb') as f: + self.assertEqual(f.read().strip(), b"this is file A") + + def test_mkdir(self): + P = self.cls(self.base) + p = P / 'newdirA' + self.assertFalse(p.exists()) + p.mkdir() + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + with self.assertRaises(OSError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + + def test_mkdir_parents(self): + # Creating a chain of directories. + p = self.cls(self.base, 'newdirB', 'newdirC') + self.assertFalse(p.exists()) + with self.assertRaises(OSError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.ENOENT) + p.mkdir(parents=True) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + with self.assertRaises(OSError) as cm: + p.mkdir(parents=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + # Test `mode` arg. + mode = stat.S_IMODE(p.stat().st_mode) # Default mode. + p = self.cls(self.base, 'newdirD', 'newdirE') + p.mkdir(0o555, parents=True) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + if os.name != 'nt': + # The directory's permissions follow the mode argument. + self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o7555 & mode) + # The parent's permissions follow the default process settings. + self.assertEqual(stat.S_IMODE(p.parent.stat().st_mode), mode) + + def test_mkdir_exist_ok(self): + p = self.cls(self.base, 'dirB') + st_ctime_first = p.stat().st_ctime + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + with self.assertRaises(FileExistsError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + p.mkdir(exist_ok=True) + self.assertTrue(p.exists()) + self.assertEqual(p.stat().st_ctime, st_ctime_first) + + def test_mkdir_exist_ok_with_parent(self): + p = self.cls(self.base, 'dirC') + self.assertTrue(p.exists()) + with self.assertRaises(FileExistsError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + p = p / 'newdirC' + p.mkdir(parents=True) + st_ctime_first = p.stat().st_ctime + self.assertTrue(p.exists()) + with self.assertRaises(FileExistsError) as cm: + p.mkdir(parents=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + p.mkdir(parents=True, exist_ok=True) + self.assertTrue(p.exists()) + self.assertEqual(p.stat().st_ctime, st_ctime_first) + + @unittest.skipIf(is_emscripten, "FS root cannot be modified on Emscripten.") + def test_mkdir_exist_ok_root(self): + # Issue #25803: A drive root could raise PermissionError on Windows. + self.cls('/').resolve().mkdir(exist_ok=True) + self.cls('/').resolve().mkdir(parents=True, exist_ok=True) + + @only_nt # XXX: not sure how to test this on POSIX. + def test_mkdir_with_unknown_drive(self): + for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': + p = self.cls(d + ':\\') + if not p.is_dir(): + break + else: + self.skipTest("cannot find a drive that doesn't exist") + with self.assertRaises(OSError): + (p / 'child' / 'path').mkdir(parents=True) + + def test_mkdir_with_child_file(self): + p = self.cls(self.base, 'dirB', 'fileB') + self.assertTrue(p.exists()) + # An exception is raised when the last path component is an existing + # regular file, regardless of whether exist_ok is true or not. + with self.assertRaises(FileExistsError) as cm: + p.mkdir(parents=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(FileExistsError) as cm: + p.mkdir(parents=True, exist_ok=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + + def test_mkdir_no_parents_file(self): + p = self.cls(self.base, 'fileA') + self.assertTrue(p.exists()) + # An exception is raised when the last path component is an existing + # regular file, regardless of whether exist_ok is true or not. + with self.assertRaises(FileExistsError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(FileExistsError) as cm: + p.mkdir(exist_ok=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + + def test_mkdir_concurrent_parent_creation(self): + for pattern_num in range(32): + p = self.cls(self.base, 'dirCPC%d' % pattern_num) + self.assertFalse(p.exists()) + + real_mkdir = os.mkdir + def my_mkdir(path, mode=0o777): + path = str(path) + # Emulate another process that would create the directory + # just before we try to create it ourselves. We do it + # in all possible pattern combinations, assuming that this + # function is called at most 5 times (dirCPC/dir1/dir2, + # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). + if pattern.pop(): + real_mkdir(path, mode) # From another process. + concurrently_created.add(path) + real_mkdir(path, mode) # Our real call. + + pattern = [bool(pattern_num & (1 << n)) for n in range(5)] + concurrently_created = set() + p12 = p / 'dir1' / 'dir2' + try: + with mock.patch("os.mkdir", my_mkdir): + p12.mkdir(parents=True, exist_ok=False) + except FileExistsError: + self.assertIn(str(p12), concurrently_created) + else: + self.assertNotIn(str(p12), concurrently_created) + self.assertTrue(p.exists()) + + def test_symlink_to(self): + if not self.can_symlink: + self.skipTest("symlinks required") + P = self.cls(self.base) + target = P / 'fileA' + # Symlinking a path target. + link = P / 'dirA' / 'linkAA' + link.symlink_to(target) + self.assertEqual(link.stat(), target.stat()) + self.assertNotEqual(link.lstat(), target.stat()) + # Symlinking a str target. + link = P / 'dirA' / 'linkAAA' + link.symlink_to(str(target)) + self.assertEqual(link.stat(), target.stat()) + self.assertNotEqual(link.lstat(), target.stat()) + self.assertFalse(link.is_dir()) + # Symlinking to a directory. + target = P / 'dirB' + link = P / 'dirA' / 'linkAAAA' + link.symlink_to(target, target_is_directory=True) + self.assertEqual(link.stat(), target.stat()) + self.assertNotEqual(link.lstat(), target.stat()) + self.assertTrue(link.is_dir()) + self.assertTrue(list(link.iterdir())) + + @unittest.skipIf(hasattr(os, "symlink"), "os.symlink() is present") + def test_symlink_to_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + # linking to another path. + q = P / 'dirA' / 'fileAA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.symlink_to(p) + + def test_is_junction(self): + P = self.cls(self.base) + + with mock.patch.object(P.pathmod, 'isjunction'): + self.assertEqual(P.is_junction(), P.pathmod.isjunction.return_value) + P.pathmod.isjunction.assert_called_once_with(P) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_is_fifo_true(self): + P = self.cls(self.base, 'myfifo') + try: + os.mkfifo(str(P)) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) + self.assertTrue(P.is_fifo()) + self.assertFalse(P.is_socket()) + self.assertFalse(P.is_file()) + self.assertIs(self.cls(self.base, 'myfifo\udfff').is_fifo(), False) + self.assertIs(self.cls(self.base, 'myfifo\x00').is_fifo(), False) + + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") + @unittest.skipIf( + is_emscripten, "Unix sockets are not implemented on Emscripten." + ) + @unittest.skipIf( + is_wasi, "Cannot create socket on WASI." + ) + def test_is_socket_true(self): + P = self.cls(self.base, 'mysock') + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.addCleanup(sock.close) + try: + sock.bind(str(P)) + except OSError as e: + if (isinstance(e, PermissionError) or + "AF_UNIX path too long" in str(e)): + self.skipTest("cannot bind Unix socket: " + str(e)) + self.assertTrue(P.is_socket()) + self.assertFalse(P.is_fifo()) + self.assertFalse(P.is_file()) + self.assertIs(self.cls(self.base, 'mysock\udfff').is_socket(), False) + self.assertIs(self.cls(self.base, 'mysock\x00').is_socket(), False) + + def test_is_char_device_true(self): + # Under Unix, /dev/null should generally be a char device. + P = self.cls('/dev/null') + if not P.exists(): + self.skipTest("/dev/null required") + self.assertTrue(P.is_char_device()) + self.assertFalse(P.is_block_device()) + self.assertFalse(P.is_file()) + self.assertIs(self.cls('/dev/null\udfff').is_char_device(), False) + self.assertIs(self.cls('/dev/null\x00').is_char_device(), False) + + def test_is_mount_root(self): + if os.name == 'nt': + R = self.cls('c:\\') + else: + R = self.cls('/') + self.assertTrue(R.is_mount()) + self.assertFalse((R / '\udfff').is_mount()) + + def test_passing_kwargs_deprecated(self): + with self.assertWarns(DeprecationWarning): + self.cls(foo="bar") + + def setUpWalk(self): + super().setUpWalk() + sub21_path= self.sub2_path / "SUB21" + tmp5_path = sub21_path / "tmp3" + broken_link3_path = self.sub2_path / "broken_link3" + + os.makedirs(sub21_path) + tmp5_path.write_text("I am tmp5, blame test_pathlib.") + if self.can_symlink: + os.symlink(tmp5_path, broken_link3_path) + self.sub2_tree[2].append('broken_link3') + self.sub2_tree[2].sort() + if not is_emscripten: + # Emscripten fails with inaccessible directories. + os.chmod(sub21_path, 0) + try: + os.listdir(sub21_path) + except PermissionError: + self.sub2_tree[1].append('SUB21') + else: + os.chmod(sub21_path, stat.S_IRWXU) + os.unlink(tmp5_path) + os.rmdir(sub21_path) + + def test_walk_bad_dir(self): + self.setUpWalk() + errors = [] + walk_it = self.walk_path.walk(on_error=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(errors, []) + dir1 = 'SUB1' + path1 = root / dir1 + path1new = (root / dir1).with_suffix(".new") + path1.rename(path1new) + try: + roots = [r for r, _, _ in walk_it] + self.assertTrue(errors) + self.assertNotIn(path1, roots) + self.assertNotIn(path1new, roots) + for dir2 in dirs: + if dir2 != dir1: + self.assertIn(root / dir2, roots) + finally: + path1new.rename(path1) + + def test_walk_many_open_files(self): + depth = 30 + base = self.cls(self.base, 'deep') + path = self.cls(base, *(['d']*depth)) + path.mkdir(parents=True) + + iters = [base.walk(top_down=False) for _ in range(100)] + for i in range(depth + 1): + expected = (path, ['d'] if i else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path.parent + + iters = [base.walk(top_down=True) for _ in range(100)] + path = base + for i in range(depth + 1): + expected = (path, ['d'] if i < depth else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path / 'd' + + def test_walk_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with set_recursion_limit(recursion_limit): + list(base.walk()) + list(base.walk(top_down=False)) + + def test_glob_many_open_files(self): + depth = 30 + P = self.cls + p = base = P(self.base) / 'deep' + p.mkdir() + for _ in range(depth): + p /= 'd' + p.mkdir() + pattern = '/'.join(['*'] * depth) + iters = [base.glob(pattern) for j in range(100)] + for it in iters: + self.assertEqual(next(it), p) + iters = [base.rglob('d') for j in range(100)] + p = base + for i in range(depth): + p = p / 'd' + for it in iters: + self.assertEqual(next(it), p) + + def test_glob_above_recursion_limit(self): + recursion_limit = 50 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with set_recursion_limit(recursion_limit): + list(base.glob('**/')) + + +@only_posix +class PosixPathTest(PathTest, PurePosixPathTest): + cls = pathlib.PosixPath + + def test_absolute(self): + P = self.cls + self.assertEqual(str(P('/').absolute()), '/') + self.assertEqual(str(P('/a').absolute()), '/a') + self.assertEqual(str(P('/a/b').absolute()), '/a/b') + + # '//'-prefixed absolute path (supported by POSIX). + self.assertEqual(str(P('//').absolute()), '//') + self.assertEqual(str(P('//a').absolute()), '//a') + self.assertEqual(str(P('//a/b').absolute()), '//a/b') + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + def test_open_mode(self): + old_mask = os.umask(0) + self.addCleanup(os.umask, old_mask) + p = self.cls(self.base) + with (p / 'new_file').open('wb'): + pass + st = os.stat(self.pathmod.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) + os.umask(0o022) + with (p / 'other_new_file').open('wb'): + pass + st = os.stat(self.pathmod.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + + def test_resolve_root(self): + current_directory = os.getcwd() + try: + os.chdir('/') + p = self.cls('spam') + self.assertEqual(str(p.resolve()), '/spam') + finally: + os.chdir(current_directory) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + def test_touch_mode(self): + old_mask = os.umask(0) + self.addCleanup(os.umask, old_mask) + p = self.cls(self.base) + (p / 'new_file').touch() + st = os.stat(self.pathmod.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) + os.umask(0o022) + (p / 'other_new_file').touch() + st = os.stat(self.pathmod.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + (p / 'masked_new_file').touch(mode=0o750) + st = os.stat(self.pathmod.join(self.base, 'masked_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) + + def test_glob(self): + P = self.cls + p = P(self.base) + given = set(p.glob("FILEa")) + expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given + self.assertEqual(given, expect) + self.assertEqual(set(p.glob("FILEa*")), set()) + + def test_rglob(self): + P = self.cls + p = P(self.base, "dirC") + given = set(p.rglob("FILEd")) + expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given + self.assertEqual(given, expect) + self.assertEqual(set(p.rglob("FILEd*")), set()) + + @unittest.skipUnless(hasattr(pwd, 'getpwall'), + 'pwd module does not expose getpwall()') + @unittest.skipIf(sys.platform == "vxworks", + "no home directory on VxWorks") + def test_expanduser(self): + P = self.cls + import_helper.import_module('pwd') + import pwd + pwdent = pwd.getpwuid(os.getuid()) + username = pwdent.pw_name + userhome = pwdent.pw_dir.rstrip('/') or '/' + # Find arbitrary different user (if exists). + for pwdent in pwd.getpwall(): + othername = pwdent.pw_name + otherhome = pwdent.pw_dir.rstrip('/') + if othername != username and otherhome: + break + else: + othername = username + otherhome = userhome + + fakename = 'fakeuser' + # This user can theoretically exist on a test runner. Create unique name: + try: + while pwd.getpwnam(fakename): + fakename += '1' + except KeyError: + pass # Non-existent name found + + p1 = P('~/Documents') + p2 = P(f'~{username}/Documents') + p3 = P(f'~{othername}/Documents') + p4 = P(f'../~{username}/Documents') + p5 = P(f'/~{username}/Documents') + p6 = P('') + p7 = P(f'~{fakename}/Documents') + + with os_helper.EnvironmentVarGuard() as env: + env.pop('HOME', None) + + self.assertEqual(p1.expanduser(), P(userhome) / 'Documents') + self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') + self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + self.assertRaises(RuntimeError, p7.expanduser) + + env['HOME'] = '/tmp' + self.assertEqual(p1.expanduser(), P('/tmp/Documents')) + self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') + self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + self.assertRaises(RuntimeError, p7.expanduser) + + @unittest.skipIf(sys.platform != "darwin", + "Bad file descriptor in /dev/fd affects only macOS") + def test_handling_bad_descriptor(self): + try: + file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:] + if not file_descriptors: + self.skipTest("no file descriptors - issue was not reproduced") + # Checking all file descriptors because there is no guarantee + # which one will fail. + for f in file_descriptors: + f.exists() + f.is_dir() + f.is_file() + f.is_symlink() + f.is_block_device() + f.is_char_device() + f.is_fifo() + f.is_socket() + except OSError as e: + if e.errno == errno.EBADF: + self.fail("Bad file descriptor not handled.") + raise + + def test_from_uri(self): + P = self.cls + self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar')) + self.assertEqual(P.from_uri('file://foo/bar'), P('//foo/bar')) + self.assertEqual(P.from_uri('file:///foo/bar'), P('/foo/bar')) + self.assertEqual(P.from_uri('file:////foo/bar'), P('//foo/bar')) + self.assertEqual(P.from_uri('file://localhost/foo/bar'), P('/foo/bar')) + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, '/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') + + def test_from_uri_pathname2url(self): + P = self.cls + self.assertEqual(P.from_uri('file:' + pathname2url('/foo/bar')), P('/foo/bar')) + self.assertEqual(P.from_uri('file:' + pathname2url('//foo/bar')), P('//foo/bar')) + + +@only_nt +class WindowsPathTest(PathTest, PureWindowsPathTest): + cls = pathlib.WindowsPath + + def test_absolute(self): + P = self.cls + + # Simple absolute paths. + self.assertEqual(str(P('c:\\').absolute()), 'c:\\') + self.assertEqual(str(P('c:\\a').absolute()), 'c:\\a') + self.assertEqual(str(P('c:\\a\\b').absolute()), 'c:\\a\\b') + + # UNC absolute paths. + share = '\\\\server\\share\\' + self.assertEqual(str(P(share).absolute()), share) + self.assertEqual(str(P(share + 'a').absolute()), share + 'a') + self.assertEqual(str(P(share + 'a\\b').absolute()), share + 'a\\b') + + # UNC relative paths. + with mock.patch("os.getcwd") as getcwd: + getcwd.return_value = share + + self.assertEqual(str(P().absolute()), share) + self.assertEqual(str(P('.').absolute()), share) + self.assertEqual(str(P('a').absolute()), os.path.join(share, 'a')) + self.assertEqual(str(P('a', 'b', 'c').absolute()), + os.path.join(share, 'a', 'b', 'c')) + + drive = os.path.splitdrive(self.base)[0] + with os_helper.change_cwd(self.base): + # Relative path with root + self.assertEqual(str(P('\\').absolute()), drive + '\\') + self.assertEqual(str(P('\\foo').absolute()), drive + '\\foo') + + # Relative path on current drive + self.assertEqual(str(P(drive).absolute()), self.base) + self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(self.base, 'foo')) + + with os_helper.subst_drive(self.base) as other_drive: + # Set the working directory on the substitute drive + saved_cwd = os.getcwd() + other_cwd = f'{other_drive}\\dirA' + os.chdir(other_cwd) + os.chdir(saved_cwd) + + # Relative path on another drive + self.assertEqual(str(P(other_drive).absolute()), other_cwd) + self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') + + def test_glob(self): + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) + self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) + self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) + self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) + self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) + + def test_rglob(self): + P = self.cls + p = P(self.base, "dirC") + self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) + self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) + self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) + + def test_expanduser(self): + P = self.cls + with os_helper.EnvironmentVarGuard() as env: + env.pop('HOME', None) + env.pop('USERPROFILE', None) + env.pop('HOMEPATH', None) + env.pop('HOMEDRIVE', None) + env['USERNAME'] = 'alice' + + # test that the path returns unchanged + p1 = P('~/My Documents') + p2 = P('~alice/My Documents') + p3 = P('~bob/My Documents') + p4 = P('/~/My Documents') + p5 = P('d:~/My Documents') + p6 = P('') + self.assertRaises(RuntimeError, p1.expanduser) + self.assertRaises(RuntimeError, p2.expanduser) + self.assertRaises(RuntimeError, p3.expanduser) + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + + def check(): + env.pop('USERNAME', None) + self.assertEqual(p1.expanduser(), + P('C:/Users/alice/My Documents')) + self.assertRaises(RuntimeError, p2.expanduser) + env['USERNAME'] = 'alice' + self.assertEqual(p2.expanduser(), + P('C:/Users/alice/My Documents')) + self.assertEqual(p3.expanduser(), + P('C:/Users/bob/My Documents')) + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + + env['HOMEPATH'] = 'C:\\Users\\alice' + check() + + env['HOMEDRIVE'] = 'C:\\' + env['HOMEPATH'] = 'Users\\alice' + check() + + env.pop('HOMEDRIVE', None) + env.pop('HOMEPATH', None) + env['USERPROFILE'] = 'C:\\Users\\alice' + check() + + # bpo-38883: ignore `HOME` when set on windows + env['HOME'] = 'C:\\Users\\eve' + check() + + def test_from_uri(self): + P = self.cls + # DOS drive paths + self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:/c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:///c|/path/to/file'), P('c:/path/to/file')) + # UNC paths + self.assertEqual(P.from_uri('file://server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file:////server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file://///server/path/to/file'), P('//server/path/to/file')) + # Localhost paths + self.assertEqual(P.from_uri('file://localhost/c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file://localhost/c|/path/to/file'), P('c:/path/to/file')) + # Invalid paths + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, 'c:/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') + + def test_from_uri_pathname2url(self): + P = self.cls + self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file')) + + def test_owner(self): + P = self.cls + with self.assertRaises(pathlib.UnsupportedOperation): + P('c:/').owner() + + def test_group(self): + P = self.cls + with self.assertRaises(pathlib.UnsupportedOperation): + P('c:/').group() + + +class PathSubclassTest(PathTest): + class cls(pathlib.Path): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + +class CompatiblePathTest(unittest.TestCase): + """ + Test that a type can be made compatible with PurePath + derivatives by implementing division operator overloads. + """ + + class CompatPath: + """ + Minimum viable class to test PurePath compatibility. + Simply uses the division operator to join a given + string and the string value of another object with + a forward slash. + """ + def __init__(self, string): + self.string = string + + def __truediv__(self, other): + return type(self)(f"{self.string}/{other}") + + def __rtruediv__(self, other): + return type(self)(f"{other}/{self.string}") + + def test_truediv(self): + result = pathlib.PurePath("test") / self.CompatPath("right") + self.assertIsInstance(result, self.CompatPath) + self.assertEqual(result.string, "test/right") + + with self.assertRaises(TypeError): + # Verify improper operations still raise a TypeError + pathlib.PurePath("test") / 10 + + def test_rtruediv(self): + result = self.CompatPath("left") / pathlib.PurePath("test") + self.assertIsInstance(result, self.CompatPath) + self.assertEqual(result.string, "left/test") + + with self.assertRaises(TypeError): + # Verify improper operations still raise a TypeError + 10 / pathlib.PurePath("test") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py new file mode 100644 index 00000000000000..e4a4e81e547cd1 --- /dev/null +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -0,0 +1,1819 @@ +import collections.abc +import io +import os +import errno +import stat +import unittest + +from pathlib._abc import UnsupportedOperation, PurePathBase, PathBase +import posixpath + +from test.support.os_helper import TESTFN + + +class UnsupportedOperationTest(unittest.TestCase): + def test_is_notimplemented(self): + self.assertTrue(issubclass(UnsupportedOperation, NotImplementedError)) + self.assertTrue(isinstance(UnsupportedOperation(), NotImplementedError)) + + +# +# Tests for the pure classes. +# + + +class PurePathBaseTest(unittest.TestCase): + cls = PurePathBase + + def test_magic_methods(self): + P = self.cls + self.assertFalse(hasattr(P, '__fspath__')) + self.assertFalse(hasattr(P, '__bytes__')) + self.assertIs(P.__reduce__, object.__reduce__) + self.assertIs(P.__repr__, object.__repr__) + self.assertIs(P.__hash__, object.__hash__) + self.assertIs(P.__eq__, object.__eq__) + self.assertIs(P.__lt__, object.__lt__) + self.assertIs(P.__le__, object.__le__) + self.assertIs(P.__gt__, object.__gt__) + self.assertIs(P.__ge__, object.__ge__) + + def test_pathmod(self): + self.assertIs(self.cls.pathmod, posixpath) + + +class DummyPurePath(PurePathBase): + def __eq__(self, other): + if not isinstance(other, DummyPurePath): + return NotImplemented + return str(self) == str(other) + + def __hash__(self): + return hash(str(self)) + + +class DummyPurePathTest(unittest.TestCase): + cls = DummyPurePath + + # Use a base path that's unrelated to any real filesystem path. + base = f'/this/path/kills/fascists/{TESTFN}' + + # Keys are canonical paths, values are list of tuples of arguments + # supposed to produce equal paths. + equivalences = { + 'a/b': [ + ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), + ('a/b/',), ('a//b',), ('a//b//',), + # Empty components get removed. + ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), + ], + '/b/c/d': [ + ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), + # Empty components get removed. + ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), + ], + } + + def setUp(self): + p = self.cls('a') + self.pathmod = p.pathmod + self.sep = self.pathmod.sep + self.altsep = self.pathmod.altsep + + def test_constructor_common(self): + P = self.cls + p = P('a') + self.assertIsInstance(p, P) + P('a', 'b', 'c') + P('/a', 'b', 'c') + P('a/b/c') + P('/a/b/c') + + def _check_str_subclass(self, *args): + # Issue #21127: it should be possible to construct a PurePath object + # from a str subclass instance, and it then gets converted to + # a pure str object. + class StrSubclass(str): + pass + P = self.cls + p = P(*(StrSubclass(x) for x in args)) + self.assertEqual(p, P(*args)) + for part in p.parts: + self.assertIs(type(part), str) + + def test_str_subclass_common(self): + self._check_str_subclass('') + self._check_str_subclass('.') + self._check_str_subclass('a') + self._check_str_subclass('a/b.txt') + self._check_str_subclass('/a/b.txt') + + def test_with_segments_common(self): + class P(self.cls): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + p = P('foo', 'bar', session_id=42) + self.assertEqual(42, (p / 'foo').session_id) + self.assertEqual(42, ('foo' / p).session_id) + self.assertEqual(42, p.joinpath('foo').session_id) + self.assertEqual(42, p.with_name('foo').session_id) + self.assertEqual(42, p.with_stem('foo').session_id) + self.assertEqual(42, p.with_suffix('.foo').session_id) + self.assertEqual(42, p.with_segments('foo').session_id) + self.assertEqual(42, p.relative_to('foo').session_id) + self.assertEqual(42, p.parent.session_id) + for parent in p.parents: + self.assertEqual(42, parent.session_id) + + def _check_parse_path(self, raw_path, *expected): + sep = self.pathmod.sep + actual = self.cls._parse_path(raw_path.replace('/', sep)) + self.assertEqual(actual, expected) + if altsep := self.pathmod.altsep: + actual = self.cls._parse_path(raw_path.replace('/', altsep)) + self.assertEqual(actual, expected) + + def test_parse_path_common(self): + check = self._check_parse_path + sep = self.pathmod.sep + check('', '', '', []) + check('a', '', '', ['a']) + check('a/', '', '', ['a']) + check('a/b', '', '', ['a', 'b']) + check('a/b/', '', '', ['a', 'b']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b//c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('.', '', '', []) + check('././b', '', '', ['b']) + check('a/./b', '', '', ['a', 'b']) + check('a/./.', '', '', ['a']) + check('/a/b', '', sep, ['a', 'b']) + + def test_join_common(self): + P = self.cls + p = P('a/b') + pp = p.joinpath('c') + self.assertEqual(pp, P('a/b/c')) + self.assertIs(type(pp), type(p)) + pp = p.joinpath('c', 'd') + self.assertEqual(pp, P('a/b/c/d')) + pp = p.joinpath('/c') + self.assertEqual(pp, P('/c')) + + def test_div_common(self): + # Basically the same as joinpath(). + P = self.cls + p = P('a/b') + pp = p / 'c' + self.assertEqual(pp, P('a/b/c')) + self.assertIs(type(pp), type(p)) + pp = p / 'c/d' + self.assertEqual(pp, P('a/b/c/d')) + pp = p / 'c' / 'd' + self.assertEqual(pp, P('a/b/c/d')) + pp = 'c' / p / 'd' + self.assertEqual(pp, P('c/a/b/d')) + pp = p/ '/c' + self.assertEqual(pp, P('/c')) + + def _check_str(self, expected, args): + p = self.cls(*args) + self.assertEqual(str(p), expected.replace('/', self.sep)) + + def test_str_common(self): + # Canonicalized paths roundtrip. + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + self._check_str(pathstr, (pathstr,)) + # Special case for the empty path. + self._check_str('.', ('',)) + # Other tests for str() are in test_equivalences(). + + def test_as_posix_common(self): + P = self.cls + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + self.assertEqual(P(pathstr).as_posix(), pathstr) + # Other tests for as_posix() are in test_equivalences(). + + def test_eq_common(self): + P = self.cls + self.assertEqual(P('a/b'), P('a/b')) + self.assertEqual(P('a/b'), P('a', 'b')) + self.assertNotEqual(P('a/b'), P('a')) + self.assertNotEqual(P('a/b'), P('/a/b')) + self.assertNotEqual(P('a/b'), P()) + self.assertNotEqual(P('/a/b'), P('/')) + self.assertNotEqual(P(), P('/')) + self.assertNotEqual(P(), "") + self.assertNotEqual(P(), {}) + self.assertNotEqual(P(), int) + + def test_match_common(self): + P = self.cls + self.assertRaises(ValueError, P('a').match, '') + self.assertRaises(ValueError, P('a').match, '.') + # Simple relative pattern. + self.assertTrue(P('b.py').match('b.py')) + self.assertTrue(P('a/b.py').match('b.py')) + self.assertTrue(P('/a/b.py').match('b.py')) + self.assertFalse(P('a.py').match('b.py')) + self.assertFalse(P('b/py').match('b.py')) + self.assertFalse(P('/a.py').match('b.py')) + self.assertFalse(P('b.py/c').match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').match('*.py')) + self.assertTrue(P('a/b.py').match('*.py')) + self.assertTrue(P('/a/b.py').match('*.py')) + self.assertFalse(P('b.pyc').match('*.py')) + self.assertFalse(P('b./py').match('*.py')) + self.assertFalse(P('b.py/c').match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').match('a*/*.py')) + self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) + self.assertFalse(P('a.py').match('a*/*.py')) + self.assertFalse(P('/dab/c.py').match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').match('/*.py')) + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('a/b.py').match('/*.py')) + self.assertFalse(P('/a/b.py').match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').match('/a/*.py')) + self.assertFalse(P('/ab.py').match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertTrue(P('a').match('**')) + self.assertTrue(P('c.py').match('**')) + self.assertTrue(P('a/b/c.py').match('**')) + self.assertTrue(P('/a/b/c.py').match('**')) + self.assertTrue(P('/a/b/c.py').match('/**')) + self.assertTrue(P('/a/b/c.py').match('**/')) + self.assertTrue(P('/a/b/c.py').match('/a/**')) + self.assertTrue(P('/a/b/c.py').match('**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/a/b/**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/**/**/**/**/*.py')) + self.assertFalse(P('c.py').match('**/a.py')) + self.assertFalse(P('c.py').match('c/**')) + self.assertFalse(P('a/b/c.py').match('**/a')) + self.assertFalse(P('a/b/c.py').match('**/a/b')) + self.assertFalse(P('a/b/c.py').match('**/a/b/c')) + self.assertFalse(P('a/b/c.py').match('**/a/b/c.')) + self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').match('/a/b/c.py/**')) + self.assertFalse(P('a/b/c.py').match('/**/a/b/c.py')) + self.assertRaises(ValueError, P('a').match, '**a/b/c') + self.assertRaises(ValueError, P('a').match, 'a/b/c**') + # Case-sensitive flag + self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P().match('*')) + self.assertTrue(P().match('**')) + self.assertFalse(P().match('**/*')) + + def test_parts_common(self): + # `parts` returns a tuple. + sep = self.sep + P = self.cls + p = P('a/b') + parts = p.parts + self.assertEqual(parts, ('a', 'b')) + # When the path is absolute, the anchor is a separate part. + p = P('/a/b') + parts = p.parts + self.assertEqual(parts, (sep, 'a', 'b')) + + def test_equivalences(self): + for k, tuples in self.equivalences.items(): + canon = k.replace('/', self.sep) + posix = k.replace(self.sep, '/') + if canon != posix: + tuples = tuples + [ + tuple(part.replace('/', self.sep) for part in t) + for t in tuples + ] + tuples.append((posix, )) + pcanon = self.cls(canon) + for t in tuples: + p = self.cls(*t) + self.assertEqual(p, pcanon, "failed with args {}".format(t)) + self.assertEqual(hash(p), hash(pcanon)) + self.assertEqual(str(p), canon) + self.assertEqual(p.as_posix(), posix) + + def test_parent_common(self): + # Relative + P = self.cls + p = P('a/b/c') + self.assertEqual(p.parent, P('a/b')) + self.assertEqual(p.parent.parent, P('a')) + self.assertEqual(p.parent.parent.parent, P()) + self.assertEqual(p.parent.parent.parent.parent, P()) + # Anchored + p = P('/a/b/c') + self.assertEqual(p.parent, P('/a/b')) + self.assertEqual(p.parent.parent, P('/a')) + self.assertEqual(p.parent.parent.parent, P('/')) + self.assertEqual(p.parent.parent.parent.parent, P('/')) + + def test_parents_common(self): + # Relative + P = self.cls + p = P('a/b/c') + par = p.parents + self.assertEqual(len(par), 3) + self.assertEqual(par[0], P('a/b')) + self.assertEqual(par[1], P('a')) + self.assertEqual(par[2], P('.')) + self.assertEqual(par[-1], P('.')) + self.assertEqual(par[-2], P('a')) + self.assertEqual(par[-3], P('a/b')) + self.assertEqual(par[0:1], (P('a/b'),)) + self.assertEqual(par[:2], (P('a/b'), P('a'))) + self.assertEqual(par[:-1], (P('a/b'), P('a'))) + self.assertEqual(par[1:], (P('a'), P('.'))) + self.assertEqual(par[::2], (P('a/b'), P('.'))) + self.assertEqual(par[::-1], (P('.'), P('a'), P('a/b'))) + self.assertEqual(list(par), [P('a/b'), P('a'), P('.')]) + with self.assertRaises(IndexError): + par[-4] + with self.assertRaises(IndexError): + par[3] + with self.assertRaises(TypeError): + par[0] = p + # Anchored + p = P('/a/b/c') + par = p.parents + self.assertEqual(len(par), 3) + self.assertEqual(par[0], P('/a/b')) + self.assertEqual(par[1], P('/a')) + self.assertEqual(par[2], P('/')) + self.assertEqual(par[-1], P('/')) + self.assertEqual(par[-2], P('/a')) + self.assertEqual(par[-3], P('/a/b')) + self.assertEqual(par[0:1], (P('/a/b'),)) + self.assertEqual(par[:2], (P('/a/b'), P('/a'))) + self.assertEqual(par[:-1], (P('/a/b'), P('/a'))) + self.assertEqual(par[1:], (P('/a'), P('/'))) + self.assertEqual(par[::2], (P('/a/b'), P('/'))) + self.assertEqual(par[::-1], (P('/'), P('/a'), P('/a/b'))) + self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')]) + with self.assertRaises(IndexError): + par[-4] + with self.assertRaises(IndexError): + par[3] + + def test_drive_common(self): + P = self.cls + self.assertEqual(P('a/b').drive, '') + self.assertEqual(P('/a/b').drive, '') + self.assertEqual(P('').drive, '') + + def test_root_common(self): + P = self.cls + sep = self.sep + self.assertEqual(P('').root, '') + self.assertEqual(P('a/b').root, '') + self.assertEqual(P('/').root, sep) + self.assertEqual(P('/a/b').root, sep) + + def test_anchor_common(self): + P = self.cls + sep = self.sep + self.assertEqual(P('').anchor, '') + self.assertEqual(P('a/b').anchor, '') + self.assertEqual(P('/').anchor, sep) + self.assertEqual(P('/a/b').anchor, sep) + + def test_name_common(self): + P = self.cls + self.assertEqual(P('').name, '') + self.assertEqual(P('.').name, '') + self.assertEqual(P('/').name, '') + self.assertEqual(P('a/b').name, 'b') + self.assertEqual(P('/a/b').name, 'b') + self.assertEqual(P('/a/b/.').name, 'b') + self.assertEqual(P('a/b.py').name, 'b.py') + self.assertEqual(P('/a/b.py').name, 'b.py') + + def test_suffix_common(self): + P = self.cls + self.assertEqual(P('').suffix, '') + self.assertEqual(P('.').suffix, '') + self.assertEqual(P('..').suffix, '') + self.assertEqual(P('/').suffix, '') + self.assertEqual(P('a/b').suffix, '') + self.assertEqual(P('/a/b').suffix, '') + self.assertEqual(P('/a/b/.').suffix, '') + self.assertEqual(P('a/b.py').suffix, '.py') + self.assertEqual(P('/a/b.py').suffix, '.py') + self.assertEqual(P('a/.hgrc').suffix, '') + self.assertEqual(P('/a/.hgrc').suffix, '') + self.assertEqual(P('a/.hg.rc').suffix, '.rc') + self.assertEqual(P('/a/.hg.rc').suffix, '.rc') + self.assertEqual(P('a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') + + def test_suffixes_common(self): + P = self.cls + self.assertEqual(P('').suffixes, []) + self.assertEqual(P('.').suffixes, []) + self.assertEqual(P('/').suffixes, []) + self.assertEqual(P('a/b').suffixes, []) + self.assertEqual(P('/a/b').suffixes, []) + self.assertEqual(P('/a/b/.').suffixes, []) + self.assertEqual(P('a/b.py').suffixes, ['.py']) + self.assertEqual(P('/a/b.py').suffixes, ['.py']) + self.assertEqual(P('a/.hgrc').suffixes, []) + self.assertEqual(P('/a/.hgrc').suffixes, []) + self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) + + def test_stem_common(self): + P = self.cls + self.assertEqual(P('').stem, '') + self.assertEqual(P('.').stem, '') + self.assertEqual(P('..').stem, '..') + self.assertEqual(P('/').stem, '') + self.assertEqual(P('a/b').stem, 'b') + self.assertEqual(P('a/b.py').stem, 'b') + self.assertEqual(P('a/.hgrc').stem, '.hgrc') + self.assertEqual(P('a/.hg.rc').stem, '.hg') + self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('a/Some name. Ending with a dot.').stem, + 'Some name. Ending with a dot.') + + def test_with_name_common(self): + P = self.cls + self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) + self.assertRaises(ValueError, P('').with_name, 'd.xml') + self.assertRaises(ValueError, P('.').with_name, 'd.xml') + self.assertRaises(ValueError, P('/').with_name, 'd.xml') + self.assertRaises(ValueError, P('a/b').with_name, '') + self.assertRaises(ValueError, P('a/b').with_name, '.') + self.assertRaises(ValueError, P('a/b').with_name, '/c') + self.assertRaises(ValueError, P('a/b').with_name, 'c/') + self.assertRaises(ValueError, P('a/b').with_name, 'c/d') + + def test_with_stem_common(self): + P = self.cls + self.assertEqual(P('a/b').with_stem('d'), P('a/d')) + self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) + self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) + self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) + self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) + self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) + self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) + self.assertRaises(ValueError, P('').with_stem, 'd') + self.assertRaises(ValueError, P('.').with_stem, 'd') + self.assertRaises(ValueError, P('/').with_stem, 'd') + self.assertRaises(ValueError, P('a/b').with_stem, '') + self.assertRaises(ValueError, P('a/b').with_stem, '.') + self.assertRaises(ValueError, P('a/b').with_stem, '/c') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') + + def test_with_suffix_common(self): + P = self.cls + self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) + self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) + # Stripping suffix. + self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) + self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('.').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + # Invalid suffix. + self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('a/b').with_suffix, '/') + self.assertRaises(ValueError, P('a/b').with_suffix, '.') + self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') + self.assertRaises(ValueError, P('a/b').with_suffix, './.d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') + + def test_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.relative_to) + self.assertRaises(TypeError, p.relative_to, b'a') + self.assertEqual(p.relative_to(P()), P('a/b')) + self.assertEqual(p.relative_to(''), P('a/b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to('a'), P('b')) + self.assertEqual(p.relative_to('a/'), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P()) + self.assertEqual(p.relative_to('a/b'), P()) + self.assertEqual(p.relative_to(P(), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P()) + self.assertEqual(p.relative_to('a/b', walk_up=True), P()) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) + # With several args. + with self.assertWarns(DeprecationWarning): + p.relative_to('a', 'b') + p.relative_to('a', 'b', walk_up=True) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + p = P('/a/b') + self.assertEqual(p.relative_to(P('/')), P('a/b')) + self.assertEqual(p.relative_to('/'), P('a/b')) + self.assertEqual(p.relative_to(P('/a')), P('b')) + self.assertEqual(p.relative_to('/a'), P('b')) + self.assertEqual(p.relative_to('/a/'), P('b')) + self.assertEqual(p.relative_to(P('/a/b')), P()) + self.assertEqual(p.relative_to('/a/b'), P()) + self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P()) + self.assertEqual(p.relative_to('/a/b', walk_up=True), P()) + self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/c')) + self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + + def test_is_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.is_relative_to) + self.assertRaises(TypeError, p.is_relative_to, b'a') + self.assertTrue(p.is_relative_to(P())) + self.assertTrue(p.is_relative_to('')) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to('a/')) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertTrue(p.is_relative_to('a/b')) + # With several args. + with self.assertWarns(DeprecationWarning): + p.is_relative_to('a', 'b') + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + p = P('/a/b') + self.assertTrue(p.is_relative_to(P('/'))) + self.assertTrue(p.is_relative_to('/')) + self.assertTrue(p.is_relative_to(P('/a'))) + self.assertTrue(p.is_relative_to('/a')) + self.assertTrue(p.is_relative_to('/a/')) + self.assertTrue(p.is_relative_to(P('/a/b'))) + self.assertTrue(p.is_relative_to('/a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/c'))) + self.assertFalse(p.is_relative_to(P('/a/b/c'))) + self.assertFalse(p.is_relative_to(P('/a/c'))) + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('a'))) + + +# +# Tests for the virtual classes. +# + +class PathBaseTest(PurePathBaseTest): + cls = PathBase + + def test_unsupported_operation(self): + P = self.cls + p = self.cls() + e = UnsupportedOperation + self.assertRaises(e, p.stat) + self.assertRaises(e, p.lstat) + self.assertRaises(e, p.exists) + self.assertRaises(e, p.samefile, 'foo') + self.assertRaises(e, p.is_dir) + self.assertRaises(e, p.is_file) + self.assertRaises(e, p.is_mount) + self.assertRaises(e, p.is_symlink) + self.assertRaises(e, p.is_block_device) + self.assertRaises(e, p.is_char_device) + self.assertRaises(e, p.is_fifo) + self.assertRaises(e, p.is_socket) + self.assertRaises(e, p.open) + self.assertRaises(e, p.read_bytes) + self.assertRaises(e, p.read_text) + self.assertRaises(e, p.write_bytes, b'foo') + self.assertRaises(e, p.write_text, 'foo') + self.assertRaises(e, p.iterdir) + self.assertRaises(e, p.glob, '*') + self.assertRaises(e, p.rglob, '*') + self.assertRaises(e, lambda: list(p.walk())) + self.assertRaises(e, p.absolute) + self.assertRaises(e, P.cwd) + self.assertRaises(e, p.expanduser) + self.assertRaises(e, p.home) + self.assertRaises(e, p.readlink) + self.assertRaises(e, p.symlink_to, 'foo') + self.assertRaises(e, p.hardlink_to, 'foo') + self.assertRaises(e, p.mkdir) + self.assertRaises(e, p.touch) + self.assertRaises(e, p.rename, 'foo') + self.assertRaises(e, p.replace, 'foo') + self.assertRaises(e, p.chmod, 0o755) + self.assertRaises(e, p.lchmod, 0o755) + self.assertRaises(e, p.unlink) + self.assertRaises(e, p.rmdir) + self.assertRaises(e, p.owner) + self.assertRaises(e, p.group) + self.assertRaises(e, p.as_uri) + + def test_as_uri_common(self): + e = UnsupportedOperation + self.assertRaises(e, self.cls().as_uri) + + def test_fspath_common(self): + self.assertRaises(TypeError, os.fspath, self.cls()) + + def test_as_bytes_common(self): + self.assertRaises(TypeError, bytes, self.cls()) + + +class DummyPathIO(io.BytesIO): + """ + Used by DummyPath to implement `open('w')` + """ + + def __init__(self, files, path): + super().__init__() + self.files = files + self.path = path + + def close(self): + self.files[self.path] = self.getvalue() + super().close() + + +class DummyPath(PathBase): + """ + Simple implementation of PathBase that keeps files and directories in + memory. + """ + _files = {} + _directories = {} + _symlinks = {} + + def __eq__(self, other): + if not isinstance(other, DummyPath): + return NotImplemented + return str(self) == str(other) + + def __hash__(self): + return hash(str(self)) + + def stat(self, *, follow_symlinks=True): + if follow_symlinks: + path = str(self.resolve()) + else: + path = str(self.parent.resolve() / self.name) + if path in self._files: + st_mode = stat.S_IFREG + elif path in self._directories: + st_mode = stat.S_IFDIR + elif path in self._symlinks: + st_mode = stat.S_IFLNK + else: + raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) + return os.stat_result((st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0)) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + if buffering != -1: + raise NotImplementedError + path_obj = self.resolve() + path = str(path_obj) + name = path_obj.name + parent = str(path_obj.parent) + if path in self._directories: + raise IsADirectoryError(errno.EISDIR, "Is a directory", path) + + text = 'b' not in mode + mode = ''.join(c for c in mode if c not in 'btU') + if mode == 'r': + if path not in self._files: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + stream = io.BytesIO(self._files[path]) + elif mode == 'w': + if parent not in self._directories: + raise FileNotFoundError(errno.ENOENT, "File not found", parent) + stream = DummyPathIO(self._files, path) + self._files[path] = b'' + self._directories[parent].add(name) + else: + raise NotImplementedError + if text: + stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) + return stream + + def iterdir(self): + path = str(self.resolve()) + if path in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) + elif path in self._directories: + return (self / name for name in self._directories[path]) + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + path = str(self.resolve()) + if path in self._directories: + if exist_ok: + return + else: + raise FileExistsError(errno.EEXIST, "File exists", path) + try: + if self.name: + self._directories[str(self.parent)].add(self.name) + self._directories[path] = set() + except KeyError: + if not parents: + raise FileNotFoundError(errno.ENOENT, "File not found", str(self.parent)) from None + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + + +class DummyPathTest(DummyPurePathTest): + """Tests for PathBase methods that use stat(), open() and iterdir().""" + + cls = DummyPath + can_symlink = False + + # (self.base) + # | + # |-- brokenLink -> non-existing + # |-- dirA + # | `-- linkC -> ../dirB + # |-- dirB + # | |-- fileB + # | `-- linkD -> ../dirB + # |-- dirC + # | |-- dirD + # | | `-- fileD + # | `-- fileC + # | `-- novel.txt + # |-- dirE # No permissions + # |-- fileA + # |-- linkA -> fileA + # |-- linkB -> dirB + # `-- brokenLinkLoop -> brokenLinkLoop + # + + def setUp(self): + super().setUp() + pathmod = self.cls.pathmod + p = self.cls(self.base) + p.mkdir(parents=True) + p.joinpath('dirA').mkdir() + p.joinpath('dirB').mkdir() + p.joinpath('dirC').mkdir() + p.joinpath('dirC', 'dirD').mkdir() + p.joinpath('dirE').mkdir() + with p.joinpath('fileA').open('wb') as f: + f.write(b"this is file A\n") + with p.joinpath('dirB', 'fileB').open('wb') as f: + f.write(b"this is file B\n") + with p.joinpath('dirC', 'fileC').open('wb') as f: + f.write(b"this is file C\n") + with p.joinpath('dirC', 'novel.txt').open('wb') as f: + f.write(b"this is a novel\n") + with p.joinpath('dirC', 'dirD', 'fileD').open('wb') as f: + f.write(b"this is file D\n") + if self.can_symlink: + p.joinpath('linkA').symlink_to('fileA') + p.joinpath('brokenLink').symlink_to('non-existing') + p.joinpath('linkB').symlink_to('dirB') + p.joinpath('dirA', 'linkC').symlink_to(pathmod.join('..', 'dirB')) + p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB')) + p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') + + def tearDown(self): + cls = self.cls + cls._files.clear() + cls._directories.clear() + cls._symlinks.clear() + + def tempdir(self): + path = self.cls(self.base).with_name('tmp-dirD') + path.mkdir() + return path + + def assertFileNotFound(self, func, *args, **kwargs): + with self.assertRaises(FileNotFoundError) as cm: + func(*args, **kwargs) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def assertEqualNormCase(self, path_a, path_b): + normcase = self.pathmod.normcase + self.assertEqual(normcase(path_a), normcase(path_b)) + + def test_samefile(self): + pathmod = self.pathmod + fileA_path = pathmod.join(self.base, 'fileA') + fileB_path = pathmod.join(self.base, 'dirB', 'fileB') + p = self.cls(fileA_path) + pp = self.cls(fileA_path) + q = self.cls(fileB_path) + self.assertTrue(p.samefile(fileA_path)) + self.assertTrue(p.samefile(pp)) + self.assertFalse(p.samefile(fileB_path)) + self.assertFalse(p.samefile(q)) + # Test the non-existent file case + non_existent = pathmod.join(self.base, 'foo') + r = self.cls(non_existent) + self.assertRaises(FileNotFoundError, p.samefile, r) + self.assertRaises(FileNotFoundError, p.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, p) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, r) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + + def test_empty_path(self): + # The empty path points to '.' + p = self.cls('') + self.assertEqual(str(p), '.') + + def test_exists(self): + P = self.cls + p = P(self.base) + self.assertIs(True, p.exists()) + self.assertIs(True, (p / 'dirA').exists()) + self.assertIs(True, (p / 'fileA').exists()) + self.assertIs(False, (p / 'fileA' / 'bah').exists()) + if self.can_symlink: + self.assertIs(True, (p / 'linkA').exists()) + self.assertIs(True, (p / 'linkB').exists()) + self.assertIs(True, (p / 'linkB' / 'fileB').exists()) + self.assertIs(False, (p / 'linkA' / 'bah').exists()) + self.assertIs(False, (p / 'brokenLink').exists()) + self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) + self.assertIs(False, (p / 'foo').exists()) + self.assertIs(False, P('/xyzzy').exists()) + self.assertIs(False, P(self.base + '\udfff').exists()) + self.assertIs(False, P(self.base + '\x00').exists()) + + def test_open_common(self): + p = self.cls(self.base) + with (p / 'fileA').open('r') as f: + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.read(), "this is file A\n") + with (p / 'fileA').open('rb') as f: + self.assertIsInstance(f, io.BufferedIOBase) + self.assertEqual(f.read().strip(), b"this is file A") + + def test_read_write_bytes(self): + p = self.cls(self.base) + (p / 'fileA').write_bytes(b'abcdefg') + self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') + # Check that trying to write str does not truncate the file. + self.assertRaises(TypeError, (p / 'fileA').write_bytes, 'somestr') + self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') + + def test_read_write_text(self): + p = self.cls(self.base) + (p / 'fileA').write_text('äbcdefg', encoding='latin-1') + self.assertEqual((p / 'fileA').read_text( + encoding='utf-8', errors='ignore'), 'bcdefg') + # Check that trying to write bytes does not truncate the file. + self.assertRaises(TypeError, (p / 'fileA').write_text, b'somebytes') + self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') + + def test_read_text_with_newlines(self): + p = self.cls(self.base) + # Check that `\n` character change nothing + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\n'), + 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r` character replaces `\n` + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\r'), + 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r\n` character replaces `\n` + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\r\n'), + 'abcde\r\nfghlk\n\rmnopq') + + def test_write_text_with_newlines(self): + p = self.cls(self.base) + # Check that `\n` character change nothing + (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde\r\nfghlk\n\rmnopq') + # Check that `\r` character replaces `\n` + (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde\r\rfghlk\r\rmnopq') + # Check that `\r\n` character replaces `\n` + (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r\n') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde\r\r\nfghlk\r\n\rmnopq') + # Check that no argument passed will change `\n` to `os.linesep` + os_linesep_byte = bytes(os.linesep, encoding='ascii') + (p / 'fileA').write_text('abcde\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') + + def test_iterdir(self): + P = self.cls + p = P(self.base) + it = p.iterdir() + paths = set(it) + expected = ['dirA', 'dirB', 'dirC', 'dirE', 'fileA'] + if self.can_symlink: + expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] + self.assertEqual(paths, { P(self.base, q) for q in expected }) + + def test_iterdir_symlink(self): + if not self.can_symlink: + self.skipTest("symlinks required") + # __iter__ on a symlink to a directory. + P = self.cls + p = P(self.base, 'linkB') + paths = set(p.iterdir()) + expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } + self.assertEqual(paths, expected) + + def test_iterdir_nodir(self): + # __iter__ on something that is not a directory. + p = self.cls(self.base, 'fileA') + with self.assertRaises(OSError) as cm: + p.iterdir() + # ENOENT or EINVAL under Windows, ENOTDIR otherwise + # (see issue #12802). + self.assertIn(cm.exception.errno, (errno.ENOTDIR, + errno.ENOENT, errno.EINVAL)) + + def test_glob_common(self): + def _check(glob, expected): + self.assertEqual(set(glob), { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + it = p.glob("fileA") + self.assertIsInstance(it, collections.abc.Iterator) + _check(it, ["fileA"]) + _check(p.glob("fileB"), []) + _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"]) + if not self.can_symlink: + _check(p.glob("*A"), ['dirA', 'fileA']) + else: + _check(p.glob("*A"), ['dirA', 'fileA', 'linkA']) + if not self.can_symlink: + _check(p.glob("*B/*"), ['dirB/fileB']) + else: + _check(p.glob("*B/*"), ['dirB/fileB', 'dirB/linkD', + 'linkB/fileB', 'linkB/linkD']) + if not self.can_symlink: + _check(p.glob("*/fileB"), ['dirB/fileB']) + else: + _check(p.glob("*/fileB"), ['dirB/fileB', 'linkB/fileB']) + if self.can_symlink: + _check(p.glob("brokenLink"), ['brokenLink']) + + if not self.can_symlink: + _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/"]) + else: + _check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + + def test_glob_empty_pattern(self): + p = self.cls() + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('')) + + def test_glob_case_sensitive(self): + P = self.cls + def _check(path, pattern, case_sensitive, expected): + actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} + expected = {str(P(self.base, q)) for q in expected} + self.assertEqual(actual, expected) + path = P(self.base) + _check(path, "DIRB/FILE*", True, []) + _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) + _check(path, "dirb/file*", True, []) + _check(path, "dirb/file*", False, ["dirB/fileB"]) + + def test_glob_follow_symlinks_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + def _check(path, glob, expected): + actual = {path for path in path.glob(glob, follow_symlinks=True) + if "linkD" not in path.parent.parts} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", []) + _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) + _check(p, "*A", ["dirA", "fileA", "linkA"]) + _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) + _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/.."]) + _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", + "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirC/dirD/.."]) + _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) + + def test_glob_no_follow_symlinks_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + def _check(path, glob, expected): + actual = {path for path in path.glob(glob, follow_symlinks=False)} + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", []) + _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) + _check(p, "*A", ["dirA", "fileA", "linkA"]) + _check(p, "*B/*", ["dirB/fileB", "dirB/linkD"]) + _check(p, "*/fileB", ["dirB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"]) + _check(p, "dir*/*/..", ["dirC/dirD/.."]) + _check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**/", ["dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) + _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) + + def test_rglob_common(self): + def _check(glob, expected): + self.assertEqual(set(glob), {P(self.base, q) for q in expected}) + P = self.cls + p = P(self.base) + it = p.rglob("fileA") + self.assertIsInstance(it, collections.abc.Iterator) + _check(it, ["fileA"]) + _check(p.rglob("fileB"), ["dirB/fileB"]) + _check(p.rglob("**/fileB"), ["dirB/fileB"]) + _check(p.rglob("*/fileA"), []) + if not self.can_symlink: + _check(p.rglob("*/fileB"), ["dirB/fileB"]) + else: + _check(p.rglob("*/fileB"), ["dirB/fileB", "dirB/linkD/fileB", + "linkB/fileB", "dirA/linkC/fileB"]) + _check(p.rglob("file*"), ["fileA", "dirB/fileB", + "dirC/fileC", "dirC/dirD/fileD"]) + if not self.can_symlink: + _check(p.rglob("*/"), [ + "dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/", + ]) + else: + _check(p.rglob("*/"), [ + "dirA/", "dirA/linkC/", "dirB/", "dirB/linkD/", "dirC/", + "dirC/dirD/", "dirE/", "linkB/", + ]) + _check(p.rglob(""), ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + + p = P(self.base, "dirC") + _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("dir*/**/"), ["dirC/dirD/"]) + _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) + _check(p.rglob("*/"), ["dirC/dirD/"]) + _check(p.rglob(""), ["dirC/", "dirC/dirD/"]) + _check(p.rglob("**/"), ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p.rglob("*.txt"), ["dirC/novel.txt"]) + _check(p.rglob("*.*"), ["dirC/novel.txt"]) + + def test_rglob_follow_symlinks_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + def _check(path, glob, expected): + actual = {path for path in path.rglob(glob, follow_symlinks=True) + if 'linkD' not in path.parent.parts} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + _check(p, "*/fileA", []) + _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", + "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) + _check(p, "", ["./", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) + + p = P(self.base, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + def test_rglob_no_follow_symlinks_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + def _check(path, glob, expected): + actual = {path for path in path.rglob(glob, follow_symlinks=False)} + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", ["dirB/fileB"]) + _check(p, "*/fileA", []) + _check(p, "*/fileB", ["dirB/fileB"]) + _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", ]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "", ["./", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + + p = P(self.base, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + def test_rglob_symlink_loop(self): + # Don't get fooled by symlink loops (Issue #26012). + if not self.can_symlink: + self.skipTest("symlinks required") + P = self.cls + p = P(self.base) + given = set(p.rglob('*')) + expect = {'brokenLink', + 'dirA', 'dirA/linkC', + 'dirB', 'dirB/fileB', 'dirB/linkD', + 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', + 'dirC/fileC', 'dirC/novel.txt', + 'dirE', + 'fileA', + 'linkA', + 'linkB', + 'brokenLinkLoop', + } + self.assertEqual(given, {p / x for x in expect}) + + def test_glob_dotdot(self): + # ".." is not special in globs. + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) + self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) + self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) + self.assertEqual(set(p.glob("dirA/../file*/..")), set()) + self.assertEqual(set(p.glob("../xyzzy")), set()) + self.assertEqual(set(p.glob("xyzzy/..")), set()) + self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + + def test_glob_permissions(self): + # See bpo-38894 + if not self.can_symlink: + self.skipTest("symlinks required") + P = self.cls + base = P(self.base) / 'permissions' + base.mkdir() + + for i in range(100): + link = base / f"link{i}" + if i % 2: + link.symlink_to(P(self.base, "dirE", "nonexistent")) + else: + link.symlink_to(P(self.base, "dirC")) + + self.assertEqual(len(set(base.glob("*"))), 100) + self.assertEqual(len(set(base.glob("*/"))), 50) + self.assertEqual(len(set(base.glob("*/fileC"))), 50) + self.assertEqual(len(set(base.glob("*/file*"))), 50) + + def test_glob_long_symlink(self): + # See gh-87695 + if not self.can_symlink: + self.skipTest("symlinks required") + base = self.cls(self.base) / 'long_symlink' + base.mkdir() + bad_link = base / 'bad_link' + bad_link.symlink_to("bad" * 200) + self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + + def test_glob_recursive_no_trailing_slash(self): + P = self.cls + p = P(self.base) + with self.assertWarns(FutureWarning): + p.glob('**') + with self.assertWarns(FutureWarning): + p.glob('*/**') + with self.assertWarns(FutureWarning): + p.rglob('**') + with self.assertWarns(FutureWarning): + p.rglob('*/**') + + + def test_readlink(self): + if not self.can_symlink: + self.skipTest("symlinks required") + P = self.cls(self.base) + self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) + self.assertEqual((P / 'brokenLink').readlink(), + self.cls('non-existing')) + self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) + self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) + with self.assertRaises(OSError): + (P / 'fileA').readlink() + + @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") + def test_readlink_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + with self.assertRaises(UnsupportedOperation): + q.readlink(p) + + def _check_resolve(self, p, expected, strict=True): + q = p.resolve(strict) + self.assertEqual(q, expected) + + # This can be used to check both relative and absolute resolutions. + _check_resolve_relative = _check_resolve_absolute = _check_resolve + + def test_resolve_common(self): + if not self.can_symlink: + self.skipTest("symlinks required") + P = self.cls + p = P(self.base, 'foo') + with self.assertRaises(OSError) as cm: + p.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ENOENT) + # Non-strict + pathmod = self.pathmod + self.assertEqualNormCase(str(p.resolve(strict=False)), + pathmod.join(self.base, 'foo')) + p = P(self.base, 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + pathmod.join(self.base, 'foo', 'in', 'spam')) + p = P(self.base, '..', 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + pathmod.join(pathmod.dirname(self.base), 'foo', 'in', 'spam')) + # These are all relative symlinks. + p = P(self.base, 'dirB', 'fileB') + self._check_resolve_relative(p, p) + p = P(self.base, 'linkA') + self._check_resolve_relative(p, P(self.base, 'fileA')) + p = P(self.base, 'dirA', 'linkC', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + p = P(self.base, 'dirB', 'linkD', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', + 'spam'), False) + p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + if self.cls.pathmod is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', + 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + # Now create absolute symlinks. + d = self.tempdir() + P(self.base, 'dirA', 'linkX').symlink_to(d) + P(self.base, str(d), 'linkY').symlink_to(self.pathmod.join(self.base, 'dirB')) + p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') + self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), + False) + p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + if self.cls.pathmod is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + + def test_resolve_dot(self): + # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ + if not self.can_symlink: + self.skipTest("symlinks required") + pathmod = self.pathmod + p = self.cls(self.base) + p.joinpath('0').symlink_to('.', target_is_directory=True) + p.joinpath('1').symlink_to(pathmod.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(pathmod.join('1', '1'), target_is_directory=True) + q = p / '2' + self.assertEqual(q.resolve(strict=True), p) + r = q / '3' / '4' + self.assertRaises(FileNotFoundError, r.resolve, strict=True) + # Non-strict + self.assertEqual(r.resolve(strict=False), p / '3' / '4') + + def _check_symlink_loop(self, *args): + path = self.cls(*args) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) + + def test_resolve_loop(self): + if not self.can_symlink: + self.skipTest("symlinks required") + if self.cls.pathmod is not posixpath: + self.skipTest("symlink loops work differently with concrete Windows paths") + # Loops with relative symlinks. + self.cls(self.base, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(self.base, 'linkX') + self.cls(self.base, 'linkY').symlink_to('linkY') + self._check_symlink_loop(self.base, 'linkY') + self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(self.base, 'linkZ') + # Non-strict + p = self.cls(self.base, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) + # Loops with absolute symlinks. + self.cls(self.base, 'linkU').symlink_to(self.pathmod.join(self.base, 'linkU/inside')) + self._check_symlink_loop(self.base, 'linkU') + self.cls(self.base, 'linkV').symlink_to(self.pathmod.join(self.base, 'linkV')) + self._check_symlink_loop(self.base, 'linkV') + self.cls(self.base, 'linkW').symlink_to(self.pathmod.join(self.base, 'linkW/../linkW')) + self._check_symlink_loop(self.base, 'linkW') + # Non-strict + q = self.cls(self.base, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) + + def test_stat(self): + statA = self.cls(self.base).joinpath('fileA').stat() + statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() + statC = self.cls(self.base).joinpath('dirC').stat() + # st_mode: files are the same, directory differs. + self.assertIsInstance(statA.st_mode, int) + self.assertEqual(statA.st_mode, statB.st_mode) + self.assertNotEqual(statA.st_mode, statC.st_mode) + self.assertNotEqual(statB.st_mode, statC.st_mode) + # st_ino: all different, + self.assertIsInstance(statA.st_ino, int) + self.assertNotEqual(statA.st_ino, statB.st_ino) + self.assertNotEqual(statA.st_ino, statC.st_ino) + self.assertNotEqual(statB.st_ino, statC.st_ino) + # st_dev: all the same. + self.assertIsInstance(statA.st_dev, int) + self.assertEqual(statA.st_dev, statB.st_dev) + self.assertEqual(statA.st_dev, statC.st_dev) + # other attributes not used by pathlib. + + def test_stat_no_follow_symlinks(self): + if not self.can_symlink: + self.skipTest("symlinks required") + p = self.cls(self.base) / 'linkA' + st = p.stat() + self.assertNotEqual(st, p.stat(follow_symlinks=False)) + + def test_stat_no_follow_symlinks_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.stat(follow_symlinks=False)) + + def test_lstat(self): + if not self.can_symlink: + self.skipTest("symlinks required") + p = self.cls(self.base)/ 'linkA' + st = p.stat() + self.assertNotEqual(st, p.lstat()) + + def test_lstat_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.lstat()) + + def test_is_dir(self): + P = self.cls(self.base) + self.assertTrue((P / 'dirA').is_dir()) + self.assertFalse((P / 'fileA').is_dir()) + self.assertFalse((P / 'non-existing').is_dir()) + self.assertFalse((P / 'fileA' / 'bah').is_dir()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_dir()) + self.assertTrue((P / 'linkB').is_dir()) + self.assertFalse((P/ 'brokenLink').is_dir()) + self.assertFalse((P / 'dirA\udfff').is_dir()) + self.assertFalse((P / 'dirA\x00').is_dir()) + + def test_is_dir_no_follow_symlinks(self): + P = self.cls(self.base) + self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_dir(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_dir(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\udfff').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) + + def test_is_file(self): + P = self.cls(self.base) + self.assertTrue((P / 'fileA').is_file()) + self.assertFalse((P / 'dirA').is_file()) + self.assertFalse((P / 'non-existing').is_file()) + self.assertFalse((P / 'fileA' / 'bah').is_file()) + if self.can_symlink: + self.assertTrue((P / 'linkA').is_file()) + self.assertFalse((P / 'linkB').is_file()) + self.assertFalse((P/ 'brokenLink').is_file()) + self.assertFalse((P / 'fileA\udfff').is_file()) + self.assertFalse((P / 'fileA\x00').is_file()) + + def test_is_file_no_follow_symlinks(self): + P = self.cls(self.base) + self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_file(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_file(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) + + def test_is_mount(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_mount()) + self.assertFalse((P / 'dirA').is_mount()) + self.assertFalse((P / 'non-existing').is_mount()) + self.assertFalse((P / 'fileA' / 'bah').is_mount()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_mount()) + + def test_is_symlink(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_symlink()) + self.assertFalse((P / 'dirA').is_symlink()) + self.assertFalse((P / 'non-existing').is_symlink()) + self.assertFalse((P / 'fileA' / 'bah').is_symlink()) + if self.can_symlink: + self.assertTrue((P / 'linkA').is_symlink()) + self.assertTrue((P / 'linkB').is_symlink()) + self.assertTrue((P/ 'brokenLink').is_symlink()) + self.assertIs((P / 'fileA\udfff').is_file(), False) + self.assertIs((P / 'fileA\x00').is_file(), False) + if self.can_symlink: + self.assertIs((P / 'linkA\udfff').is_file(), False) + self.assertIs((P / 'linkA\x00').is_file(), False) + + def test_is_junction_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_junction()) + self.assertFalse((P / 'dirA').is_junction()) + self.assertFalse((P / 'non-existing').is_junction()) + self.assertFalse((P / 'fileA' / 'bah').is_junction()) + self.assertFalse((P / 'fileA\udfff').is_junction()) + self.assertFalse((P / 'fileA\x00').is_junction()) + + def test_is_fifo_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_fifo()) + self.assertFalse((P / 'dirA').is_fifo()) + self.assertFalse((P / 'non-existing').is_fifo()) + self.assertFalse((P / 'fileA' / 'bah').is_fifo()) + self.assertIs((P / 'fileA\udfff').is_fifo(), False) + self.assertIs((P / 'fileA\x00').is_fifo(), False) + + def test_is_socket_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_socket()) + self.assertFalse((P / 'dirA').is_socket()) + self.assertFalse((P / 'non-existing').is_socket()) + self.assertFalse((P / 'fileA' / 'bah').is_socket()) + self.assertIs((P / 'fileA\udfff').is_socket(), False) + self.assertIs((P / 'fileA\x00').is_socket(), False) + + def test_is_block_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_block_device()) + self.assertFalse((P / 'dirA').is_block_device()) + self.assertFalse((P / 'non-existing').is_block_device()) + self.assertFalse((P / 'fileA' / 'bah').is_block_device()) + self.assertIs((P / 'fileA\udfff').is_block_device(), False) + self.assertIs((P / 'fileA\x00').is_block_device(), False) + + def test_is_char_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_char_device()) + self.assertFalse((P / 'dirA').is_char_device()) + self.assertFalse((P / 'non-existing').is_char_device()) + self.assertFalse((P / 'fileA' / 'bah').is_char_device()) + self.assertIs((P / 'fileA\udfff').is_char_device(), False) + self.assertIs((P / 'fileA\x00').is_char_device(), False) + + def test_parts_interning(self): + P = self.cls + p = P('/usr/bin/foo') + q = P('/usr/local/bin') + # 'usr' + self.assertIs(p.parts[1], q.parts[1]) + # 'bin' + self.assertIs(p.parts[2], q.parts[3]) + + def _check_complex_symlinks(self, link0_target): + if not self.can_symlink: + self.skipTest("symlinks required") + + # Test solving a non-looping chain of symlinks (issue #19887). + pathmod = self.pathmod + P = self.cls(self.base) + P.joinpath('link1').symlink_to(pathmod.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(pathmod.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(pathmod.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) + + # Resolve absolute paths. + p = (P / 'link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + + # Resolve relative paths. + try: + self.cls().absolute() + except UnsupportedOperation: + return + old_path = os.getcwd() + os.chdir(self.base) + try: + p = self.cls('link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + finally: + os.chdir(old_path) + + def test_complex_symlinks_absolute(self): + self._check_complex_symlinks(self.base) + + def test_complex_symlinks_relative(self): + self._check_complex_symlinks('.') + + def test_complex_symlinks_relative_dot_dot(self): + self._check_complex_symlinks(self.pathmod.join('dirA', '..')) + + def setUpWalk(self): + # Build: + # TESTFN/ + # TEST1/ a file kid and two directory kids + # tmp1 + # SUB1/ a file kid and a directory kid + # tmp2 + # SUB11/ no kids + # SUB2/ a file kid and a dirsymlink kid + # tmp3 + # link/ a symlink to TEST2 + # broken_link + # broken_link2 + # TEST2/ + # tmp4 a lone file + self.walk_path = self.cls(self.base, "TEST1") + self.sub1_path = self.walk_path / "SUB1" + self.sub11_path = self.sub1_path / "SUB11" + self.sub2_path = self.walk_path / "SUB2" + tmp1_path = self.walk_path / "tmp1" + tmp2_path = self.sub1_path / "tmp2" + tmp3_path = self.sub2_path / "tmp3" + self.link_path = self.sub2_path / "link" + t2_path = self.cls(self.base, "TEST2") + tmp4_path = self.cls(self.base, "TEST2", "tmp4") + broken_link_path = self.sub2_path / "broken_link" + broken_link2_path = self.sub2_path / "broken_link2" + + self.sub11_path.mkdir(parents=True) + self.sub2_path.mkdir(parents=True) + t2_path.mkdir(parents=True) + + for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: + with path.open("w", encoding='utf-8') as f: + f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") + + if self.can_symlink: + self.link_path.symlink_to(t2_path) + broken_link_path.symlink_to('broken') + broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) + self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) + else: + self.sub2_tree = (self.sub2_path, [], ["tmp3"]) + + def test_walk_topdown(self): + self.setUpWalk() + walker = self.walk_path.walk() + entry = next(walker) + entry[1].sort() # Ensure we visit SUB1 before SUB2 + self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"])) + entry = next(walker) + self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"])) + entry = next(walker) + self.assertEqual(entry, (self.sub11_path, [], [])) + entry = next(walker) + entry[1].sort() + entry[2].sort() + self.assertEqual(entry, self.sub2_tree) + with self.assertRaises(StopIteration): + next(walker) + + def test_walk_prune(self): + self.setUpWalk() + # Prune the search. + all = [] + for root, dirs, files in self.walk_path.walk(): + all.append((root, dirs, files)) + if 'SUB1' in dirs: + # Note that this also mutates the dirs we appended to all! + dirs.remove('SUB1') + + self.assertEqual(len(all), 2) + self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) + + all[1][-1].sort() + all[1][1].sort() + self.assertEqual(all[1], self.sub2_tree) + + def test_walk_bottom_up(self): + self.setUpWalk() + seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False + for path, dirnames, filenames in self.walk_path.walk(top_down=False): + if path == self.walk_path: + self.assertFalse(seen_testfn) + self.assertTrue(seen_sub1) + self.assertTrue(seen_sub2) + self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"]) + self.assertEqual(filenames, ["tmp1"]) + seen_testfn = True + elif path == self.sub1_path: + self.assertFalse(seen_testfn) + self.assertFalse(seen_sub1) + self.assertTrue(seen_sub11) + self.assertEqual(dirnames, ["SUB11"]) + self.assertEqual(filenames, ["tmp2"]) + seen_sub1 = True + elif path == self.sub11_path: + self.assertFalse(seen_sub1) + self.assertFalse(seen_sub11) + self.assertEqual(dirnames, []) + self.assertEqual(filenames, []) + seen_sub11 = True + elif path == self.sub2_path: + self.assertFalse(seen_testfn) + self.assertFalse(seen_sub2) + self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1])) + self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2])) + seen_sub2 = True + else: + raise AssertionError(f"Unexpected path: {path}") + self.assertTrue(seen_testfn) + + def test_walk_follow_symlinks(self): + if not self.can_symlink: + self.skipTest("symlinks required") + self.setUpWalk() + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.link_path: + self.assertEqual(dirs, []) + self.assertEqual(files, ["tmp4"]) + break + else: + self.fail("Didn't follow symlink with follow_symlinks=True") + + def test_walk_symlink_location(self): + if not self.can_symlink: + self.skipTest("symlinks required") + self.setUpWalk() + # Tests whether symlinks end up in filenames or dirnames depending + # on the `follow_symlinks` argument. + walk_it = self.walk_path.walk(follow_symlinks=False) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", files) + break + else: + self.fail("symlink not found") + + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", dirs) + break + else: + self.fail("symlink not found") + + +class DummyPathWithSymlinks(DummyPath): + def readlink(self): + path = str(self.parent.resolve() / self.name) + if path in self._symlinks: + return self.with_segments(self._symlinks[path]) + elif path in self._files or path in self._directories: + raise OSError(errno.EINVAL, "Not a symlink", path) + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def symlink_to(self, target, target_is_directory=False): + self._directories[str(self.parent)].add(self.name) + self._symlinks[str(self)] = str(target) + + +class DummyPathWithSymlinksTest(DummyPathTest): + cls = DummyPathWithSymlinks + can_symlink = True + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index dedbc828784184..298e78ccee3875 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2760,6 +2760,132 @@ def test_patma_255(self): self.assertEqual(y, 1) self.assertIs(z, x) + def test_patma_runtime_checkable_protocol(self): + # Runtime-checkable protocol + from typing import Protocol, runtime_checkable + + @runtime_checkable + class P(Protocol): + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + + def test_patma_generic_protocol(self): + # Runtime-checkable generic protocol + from typing import Generic, TypeVar, Protocol, runtime_checkable + + T = TypeVar('T') # not using PEP695 to be able to backport changes + + @runtime_checkable + class P(Protocol[T]): + a: T + b: T + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class G(Generic[T]): + def __init__(self, x: T, y: T): + self.x = x + self.y = y + + for cls in (A, G): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P(): + w = 1 + self.assertEqual(w, 0) + + def test_patma_protocol_with_match_args(self): + # Runtime-checkable protocol with `__match_args__` + from typing import Protocol, runtime_checkable + + # Used to fail before + # https://github.com/python/cpython/issues/110682 + @runtime_checkable + class P(Protocol): + __match_args__ = ('x', 'y') + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + j = 0 + match inst: + case P(x=1, y=2): + j = 1 + self.assertEqual(j, 1) + + g = 0 + match inst: + case P(x, y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + g = 1 + self.assertEqual(g, 1) + + h = 0 + match inst: + case P(1, 2): + h = 1 + self.assertEqual(h, 1) + class TestSyntaxErrors(unittest.TestCase): @@ -3198,6 +3324,35 @@ def test_class_pattern_not_type(self): w = 0 self.assertIsNone(w) + def test_regular_protocol(self): + from typing import Protocol + class P(Protocol): ... + msg = ( + 'Instance and class checks can only be used ' + 'with @runtime_checkable protocols' + ) + w = None + with self.assertRaisesRegex(TypeError, msg): + match 1: + case P(): + w = 0 + self.assertIsNone(w) + + def test_positional_patterns_with_regular_protocol(self): + from typing import Protocol + class P(Protocol): + x: int # no `__match_args__` + y: int + class A: + x = 1 + y = 2 + w = None + with self.assertRaises(TypeError): + match A(): + case P(x, y): + w = 0 + self.assertIsNone(w) + class TestValueErrors(unittest.TestCase): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 67a4053a2ac8bc..d53fe3c611bc35 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -778,6 +778,59 @@ def test_pdb_where_command(): (Pdb) continue """ +def test_pdb_interact_command(): + """Test interact command + + >>> g = 0 + >>> dict_g = {} + + >>> def test_function(): + ... x = 1 + ... lst_local = [] + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'interact', + ... 'x', + ... 'g', + ... 'x = 2', + ... 'g = 3', + ... 'dict_g["a"] = True', + ... 'lst_local.append(x)', + ... 'exit()', + ... 'p x', + ... 'p g', + ... 'p dict_g', + ... 'p lst_local', + ... 'continue', + ... ]): + ... test_function() + --Return-- + > (4)test_function()->None + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) interact + *pdb interact start* + ... x + 1 + ... g + 0 + ... x = 2 + ... g = 3 + ... dict_g["a"] = True + ... lst_local.append(x) + ... exit() + *exit from pdb interact command* + (Pdb) p x + 1 + (Pdb) p g + 0 + (Pdb) p dict_g + {'a': True} + (Pdb) p lst_local + [2] + (Pdb) continue + """ + def test_convenience_variables(): """Test convenience variables @@ -2520,15 +2573,21 @@ def tearDown(self): @unittest.skipIf(sys.flags.safe_path, 'PYTHONSAFEPATH changes default sys.path') - def _run_pdb(self, pdb_args, commands, expected_returncode=0): + def _run_pdb(self, pdb_args, commands, + expected_returncode=0, + extra_env=None): self.addCleanup(os_helper.rmtree, '__pycache__') cmd = [sys.executable, '-m', 'pdb'] + pdb_args + if extra_env is not None: + env = os.environ | extra_env + else: + env = os.environ with subprocess.Popen( cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, - env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'} + env = {**env, 'PYTHONIOENCODING': 'utf-8'} ) as proc: stdout, stderr = proc.communicate(str.encode(commands)) stdout = stdout and bytes.decode(stdout) @@ -2540,13 +2599,15 @@ def _run_pdb(self, pdb_args, commands, expected_returncode=0): ) return stdout, stderr - def run_pdb_script(self, script, commands, expected_returncode=0): + def run_pdb_script(self, script, commands, + expected_returncode=0, + extra_env=None): """Run 'script' lines with pdb and the pdb 'commands'.""" filename = 'main.py' with open(filename, 'w') as f: f.write(textwrap.dedent(script)) self.addCleanup(os_helper.unlink, filename) - return self._run_pdb([filename], commands, expected_returncode) + return self._run_pdb([filename], commands, expected_returncode, extra_env) def run_pdb_module(self, script, commands): """Runs the script code as part of a module""" @@ -3131,6 +3192,23 @@ def test_issue42384_symlink(self): self.assertEqual(stdout.split('\n')[2].rstrip('\r'), expected) + def test_safe_path(self): + """ With safe_path set, pdb should not mangle sys.path[0]""" + + script = textwrap.dedent(""" + import sys + import random + print('sys.path[0] is', sys.path[0]) + """) + commands = 'c\n' + + + with os_helper.temp_cwd() as cwd: + stdout, _ = self.run_pdb_script(script, commands, extra_env={'PYTHONSAFEPATH': '1'}) + + unexpected = f'sys.path[0] is {os.path.realpath(cwd)}' + self.assertNotIn(unexpected, stdout) + def test_issue42383(self): with os_helper.temp_cwd() as cwd: with open('foo.py', 'w') as f: @@ -3268,7 +3346,7 @@ def setUpClass(): # Ensure that the readline module is loaded # If this fails, the test is skipped because SkipTest will be raised readline = import_module('readline') - if readline.__doc__ and "libedit" in readline.__doc__: + if readline.backend == "editline": raise unittest.SkipTest("libedit readline is not supported for pdb") def test_basic_completion(self): diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index b08ababa341cfe..1d2e14a30c4e13 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -13,6 +13,8 @@ import subprocess import binascii import collections +import time +import zoneinfo from test import support from test.support import os_helper from io import BytesIO @@ -838,6 +840,54 @@ def test_xml_plist_with_entity_decl(self): "XML entity declarations are not supported"): plistlib.loads(XML_PLIST_WITH_ENTITY, fmt=plistlib.FMT_XML) + def test_load_aware_datetime(self): + dt = plistlib.loads(b"2023-12-10T08:03:30Z", + aware_datetime=True) + self.assertEqual(dt.tzinfo, datetime.UTC) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + def test_dump_utc_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, tzinfo=datetime.UTC) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_naive_datetime_with_aware_datetime_option(self): + # Save a naive datetime with aware_datetime set to true. This will lead + # to having different time as compared to the current machine's + # timezone, which is UTC. + dt = datetime.datetime(2003, 6, 7, 8, tzinfo=None) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + parsed = plistlib.loads(s, aware_datetime=False) + expected = dt.astimezone(datetime.UTC).replace(tzinfo=None) + self.assertEqual(parsed, expected) + class TestBinaryPlistlib(unittest.TestCase): @@ -962,6 +1012,28 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_load_aware_datetime(self): + data = (b'bplist003B\x04>\xd0d\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00' + b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11') + self.assertEqual(plistlib.loads(data, aware_datetime=True), + datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC)) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + class TestKeyedArchive(unittest.TestCase): def test_keyed_archive_data(self): @@ -1072,5 +1144,6 @@ def test_octal_and_hex(self): self.assertEqual(p.get("HexType"), 16777228) self.assertEqual(p.get("IntType"), 83) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 1722c84727bbd8..9c382ace806e0f 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -791,7 +791,7 @@ def check_stat(uid, gid): self.assertRaises(TypeError, chown_func, first_param, uid, t(gid)) check_stat(uid, gid) - @os_helper.skip_unless_working_chmod + @unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") @unittest.skipIf(support.is_emscripten, "getgid() is a stub") def test_chown(self): # raise an OSError if the file does not exist @@ -935,6 +935,131 @@ def test_utime(self): posix.utime(os_helper.TESTFN, (int(now), int(now))) posix.utime(os_helper.TESTFN, (now, now)) + def check_chmod(self, chmod_func, target, **kwargs): + closefd = not isinstance(target, int) + mode = os.stat(target).st_mode + try: + new_mode = mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(target, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + if stat.S_ISREG(mode): + try: + with open(target, 'wb+', closefd=closefd): + pass + except PermissionError: + pass + new_mode = mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(target, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + if stat.S_ISREG(mode): + with open(target, 'wb+', closefd=closefd): + pass + finally: + chmod_func(target, mode) + + @os_helper.skip_unless_working_chmod + def test_chmod_file(self): + self.check_chmod(posix.chmod, os_helper.TESTFN) + + def tempdir(self): + target = os_helper.TESTFN + 'd' + posix.mkdir(target) + self.addCleanup(posix.rmdir, target) + return target + + @os_helper.skip_unless_working_chmod + def test_chmod_dir(self): + target = self.tempdir() + self.check_chmod(posix.chmod, target) + + @os_helper.skip_unless_working_chmod + def test_fchmod_file(self): + with open(os_helper.TESTFN, 'wb+') as f: + self.check_chmod(posix.fchmod, f.fileno()) + self.check_chmod(posix.chmod, f.fileno()) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + def test_lchmod_file(self): + self.check_chmod(posix.lchmod, os_helper.TESTFN) + self.check_chmod(posix.chmod, os_helper.TESTFN, follow_symlinks=False) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + def test_lchmod_dir(self): + target = self.tempdir() + self.check_chmod(posix.lchmod, target) + self.check_chmod(posix.chmod, target, follow_symlinks=False) + + def check_chmod_link(self, chmod_func, target, link, **kwargs): + target_mode = os.stat(target).st_mode + link_mode = os.lstat(link).st_mode + try: + new_mode = target_mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + self.assertEqual(os.lstat(link).st_mode, link_mode) + new_mode = target_mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + self.assertEqual(os.lstat(link).st_mode, link_mode) + finally: + posix.chmod(target, target_mode) + + def check_lchmod_link(self, chmod_func, target, link, **kwargs): + target_mode = os.stat(target).st_mode + link_mode = os.lstat(link).st_mode + new_mode = link_mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, target_mode) + self.assertEqual(os.lstat(link).st_mode, new_mode) + new_mode = link_mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, target_mode) + self.assertEqual(os.lstat(link).st_mode, new_mode) + + @os_helper.skip_unless_symlink + def test_chmod_file_symlink(self): + target = os_helper.TESTFN + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + if os.name == 'nt': + self.check_lchmod_link(posix.chmod, target, link) + else: + self.check_chmod_link(posix.chmod, target, link) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + + @os_helper.skip_unless_symlink + def test_chmod_dir_symlink(self): + target = self.tempdir() + link = os_helper.TESTFN + '-link' + os.symlink(target, link, target_is_directory=True) + self.addCleanup(posix.unlink, link) + if os.name == 'nt': + self.check_lchmod_link(posix.chmod, target, link) + else: + self.check_chmod_link(posix.chmod, target, link) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + @os_helper.skip_unless_symlink + def test_lchmod_file_symlink(self): + target = os_helper.TESTFN + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + self.check_lchmod_link(posix.chmod, target, link, follow_symlinks=False) + self.check_lchmod_link(posix.lchmod, target, link) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + @os_helper.skip_unless_symlink + def test_lchmod_dir_symlink(self): + target = self.tempdir() + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + self.check_lchmod_link(posix.chmod, target, link, follow_symlinks=False) + self.check_lchmod_link(posix.lchmod, target, link) + def _test_chflags_regular_file(self, chflags_func, target_file, **kwargs): st = os.stat(target_file) self.assertTrue(hasattr(st, 'st_flags')) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 9be4640f970aef..86ce1b1d41ba61 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -47,18 +47,26 @@ def tearDown(self): safe_rmdir(os_helper.TESTFN + suffix) def test_join(self): - self.assertEqual(posixpath.join("/foo", "bar", "/bar", "baz"), - "/bar/baz") - self.assertEqual(posixpath.join("/foo", "bar", "baz"), "/foo/bar/baz") - self.assertEqual(posixpath.join("/foo/", "bar/", "baz/"), - "/foo/bar/baz/") - - self.assertEqual(posixpath.join(b"/foo", b"bar", b"/bar", b"baz"), - b"/bar/baz") - self.assertEqual(posixpath.join(b"/foo", b"bar", b"baz"), - b"/foo/bar/baz") - self.assertEqual(posixpath.join(b"/foo/", b"bar/", b"baz/"), - b"/foo/bar/baz/") + fn = posixpath.join + self.assertEqual(fn("/foo", "bar", "/bar", "baz"), "/bar/baz") + self.assertEqual(fn("/foo", "bar", "baz"), "/foo/bar/baz") + self.assertEqual(fn("/foo/", "bar/", "baz/"), "/foo/bar/baz/") + + self.assertEqual(fn(b"/foo", b"bar", b"/bar", b"baz"), b"/bar/baz") + self.assertEqual(fn(b"/foo", b"bar", b"baz"), b"/foo/bar/baz") + self.assertEqual(fn(b"/foo/", b"bar/", b"baz/"), b"/foo/bar/baz/") + + self.assertEqual(fn("a", "b"), "a/b") + self.assertEqual(fn("a", "b/"), "a/b/") + self.assertEqual(fn("a/", "b"), "a/b") + self.assertEqual(fn("a/", "b/"), "a/b/") + self.assertEqual(fn("a", "b/c", "d"), "a/b/c/d") + self.assertEqual(fn("a", "b//c", "d"), "a/b//c/d") + self.assertEqual(fn("a", "b/c/", "d"), "a/b/c/d") + self.assertEqual(fn("/a", "b"), "/a/b") + self.assertEqual(fn("/a/", "b"), "/a/b") + self.assertEqual(fn("a", "/b", "c"), "/b/c") + self.assertEqual(fn("a", "/b", "/c"), "/c") def test_split(self): self.assertEqual(posixpath.split("/foo/bar"), ("/foo", "bar")) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 745717f492e07a..99b19d01783a10 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -32,7 +32,7 @@ from test.support import threading_helper from test.support import (reap_children, captured_output, captured_stdout, captured_stderr, is_emscripten, is_wasi, - requires_docstrings) + requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) from test import pydoc_mod @@ -745,14 +745,18 @@ def test_splitdoc_with_description(self): def test_is_package_when_not_package(self): with os_helper.temp_cwd() as test_dir: - self.assertFalse(pydoc.ispackage(test_dir)) + with self.assertWarns(DeprecationWarning) as cm: + self.assertFalse(pydoc.ispackage(test_dir)) + self.assertEqual(cm.filename, __file__) def test_is_package_when_is_package(self): with os_helper.temp_cwd() as test_dir: init_path = os.path.join(test_dir, '__init__.py') open(init_path, 'w').close() - self.assertTrue(pydoc.ispackage(test_dir)) + with self.assertWarns(DeprecationWarning) as cm: + self.assertTrue(pydoc.ispackage(test_dir)) os.remove(init_path) + self.assertEqual(cm.filename, __file__) def test_allmethods(self): # issue 17476: allmethods was no longer returning unbound methods. @@ -870,6 +874,96 @@ class B(A) for expected_line in expected_lines: self.assertIn(expected_line, as_text) + def test_long_signatures(self): + from collections.abc import Callable + from typing import Literal, Annotated + + class A: + def __init__(self, + arg1: Callable[[int, int, int], str], + arg2: Literal['some value', 'other value'], + arg3: Annotated[int, 'some docs about this type'], + ) -> None: + ... + + doc = pydoc.render_doc(A) + # clean up the extra text formatting that pydoc performs + doc = re.sub('\b.', '', doc) + self.assertEqual(doc, '''Python Library Documentation: class A in module %s + +class A(builtins.object) + | A( + | arg1: collections.abc.Callable[[int, int, int], str], + | arg2: Literal['some value', 'other value'], + | arg3: Annotated[int, 'some docs about this type'] + | ) -> None + | + | Methods defined here: + | + | __init__( + | self, + | arg1: collections.abc.Callable[[int, int, int], str], + | arg2: Literal['some value', 'other value'], + | arg3: Annotated[int, 'some docs about this type'] + | ) -> None + | + | ---------------------------------------------------------------------- + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s +''' % (__name__, + '' if MISSING_C_DOCSTRINGS else '\n | dictionary for instance variables', + '' if MISSING_C_DOCSTRINGS else '\n | list of weak references to the object', + )) + + def func( + arg1: Callable[[Annotated[int, 'Some doc']], str], + arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8], + ) -> Annotated[int, 'Some other']: + ... + + doc = pydoc.render_doc(func) + # clean up the extra text formatting that pydoc performs + doc = re.sub('\b.', '', doc) + self.assertEqual(doc, '''Python Library Documentation: function func in module %s + +func( + arg1: collections.abc.Callable[[typing.Annotated[int, 'Some doc']], str], + arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8] +) -> Annotated[int, 'Some other'] +''' % __name__) + + def function_with_really_long_name_so_annotations_can_be_rather_small( + arg1: int, + arg2: str, + ): + ... + + doc = pydoc.render_doc(function_with_really_long_name_so_annotations_can_be_rather_small) + # clean up the extra text formatting that pydoc performs + doc = re.sub('\b.', '', doc) + self.assertEqual(doc, '''Python Library Documentation: function function_with_really_long_name_so_annotations_can_be_rather_small in module %s + +function_with_really_long_name_so_annotations_can_be_rather_small( + arg1: int, + arg2: str +) +''' % __name__) + + does_not_have_name = lambda \ + very_long_parameter_name_that_should_not_fit_into_a_single_line, \ + second_very_long_parameter_name: ... + + doc = pydoc.render_doc(does_not_have_name) + # clean up the extra text formatting that pydoc performs + doc = re.sub('\b.', '', doc) + self.assertEqual(doc, '''Python Library Documentation: function in module %s + + lambda very_long_parameter_name_that_should_not_fit_into_a_single_line, second_very_long_parameter_name +''' % __name__) + def test__future__imports(self): # __future__ features are excluded from module help, # except when it's the __future__ module itself @@ -1065,13 +1159,15 @@ def test_generic_alias(self): doc = pydoc.render_doc(typing.List[int], renderer=pydoc.plaintext) self.assertIn('_GenericAlias in module typing', doc) self.assertIn('List = class list(object)', doc) - self.assertIn(list.__doc__.strip().splitlines()[0], doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) self.assertEqual(pydoc.describe(list[int]), 'GenericAlias') doc = pydoc.render_doc(list[int], renderer=pydoc.plaintext) self.assertIn('GenericAlias in module builtins', doc) self.assertIn('\nclass list(object)', doc) - self.assertIn(list.__doc__.strip().splitlines()[0], doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) def test_union_type(self): self.assertEqual(pydoc.describe(typing.Union[int, str]), '_UnionGenericAlias') @@ -1085,7 +1181,8 @@ def test_union_type(self): doc = pydoc.render_doc(int | str, renderer=pydoc.plaintext) self.assertIn('UnionType in module types object', doc) self.assertIn('\nclass UnionType(builtins.object)', doc) - self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) def test_special_form(self): self.assertEqual(pydoc.describe(typing.NoReturn), '_SpecialForm') @@ -1238,6 +1335,7 @@ def test_bound_builtin_classmethod_o(self): "__class_getitem__(object, /) method of builtins.type instance") @support.cpython_only + @requires_docstrings def test_module_level_callable_unrepresentable_default(self): import _testcapi builtin = _testcapi.func_with_unrepresentable_signature @@ -1245,6 +1343,7 @@ def test_module_level_callable_unrepresentable_default(self): "func_with_unrepresentable_signature(a, b=)") @support.cpython_only + @requires_docstrings def test_builtin_staticmethod_unrepresentable_default(self): self.assertEqual(self._get_summary_line(str.maketrans), "maketrans(x, y=, z=, /)") @@ -1254,6 +1353,7 @@ def test_builtin_staticmethod_unrepresentable_default(self): "staticmeth(a, b=)") @support.cpython_only + @requires_docstrings def test_unbound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line(dict.pop), "pop(self, key, default=, /)") @@ -1263,6 +1363,7 @@ def test_unbound_builtin_method_unrepresentable_default(self): "meth(self, /, a, b=)") @support.cpython_only + @requires_docstrings def test_bound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line({}.pop), "pop(key, default=, /) " @@ -1274,6 +1375,7 @@ def test_bound_builtin_method_unrepresentable_default(self): "method of _testcapi.DocStringUnrepresentableSignatureTest instance") @support.cpython_only + @requires_docstrings def test_unbound_builtin_classmethod_unrepresentable_default(self): import _testcapi cls = _testcapi.DocStringUnrepresentableSignatureTest @@ -1282,6 +1384,7 @@ def test_unbound_builtin_classmethod_unrepresentable_default(self): "classmeth(type, /, a, b=)") @support.cpython_only + @requires_docstrings def test_bound_builtin_classmethod_unrepresentable_default(self): import _testcapi cls = _testcapi.DocStringUnrepresentableSignatureTest diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 5936d7535edd5f..6d26a61bee4292 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -185,6 +185,20 @@ def test_class_cause(self): else: self.fail("No exception raised") + def test_class_cause_nonexception_result(self): + class ConstructsNone(BaseException): + @classmethod + def __new__(*args, **kwargs): + return None + try: + raise IndexError from ConstructsNone + except TypeError as e: + self.assertIn("should have returned an instance of BaseException", str(e)) + except IndexError: + self.fail("Wrong kind of exception raised") + else: + self.fail("No exception raised") + def test_instance_cause(self): cause = KeyError() try: diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 1eca22f45378df..993a7d6e264a1f 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -47,7 +47,7 @@ def recurse(actual, expect): recurse(actual, expect) def checkPatternError(self, pattern, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(pattern) with self.subTest(pattern=pattern): err = cm.exception @@ -56,7 +56,7 @@ def checkPatternError(self, pattern, errmsg, pos=None): self.assertEqual(err.pos, pos) def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.sub(pattern, repl, string) with self.subTest(pattern=pattern, repl=repl): err = cm.exception @@ -64,6 +64,9 @@ def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): if pos is not None: self.assertEqual(err.pos, pos) + def test_error_is_PatternError_alias(self): + assert re.error is re.PatternError + def test_keep_buffer(self): # See bug 14212 b = bytearray(b'x') @@ -154,7 +157,7 @@ def test_basic_re_sub(self): (chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)+chr(8))) for c in 'cdehijklmopqsuwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ': with self.subTest(c): - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): self.assertEqual(re.sub('a', '\\' + c, 'a'), '\\' + c) self.assertEqual(re.sub(r'^\s*', 'X', 'test'), 'Xtest') @@ -836,10 +839,10 @@ def test_other_escapes(self): re.purge() # for warnings for c in 'ceghijklmopqyzCEFGHIJKLMNOPQRTVXY': with self.subTest(c): - self.assertRaises(re.error, re.compile, '\\%c' % c) + self.assertRaises(re.PatternError, re.compile, '\\%c' % c) for c in 'ceghijklmopqyzABCEFGHIJKLMNOPQRTVXYZ': with self.subTest(c): - self.assertRaises(re.error, re.compile, '[\\%c]' % c) + self.assertRaises(re.PatternError, re.compile, '[\\%c]' % c) def test_named_unicode_escapes(self): # test individual Unicode named escapes @@ -970,14 +973,14 @@ def test_lookbehind(self): self.assertIsNone(re.match(r'(?:(a)|(x))b(?<=(?(1)c|x))c', 'abc')) self.assertTrue(re.match(r'(?:(a)|(x))b(?<=(?(1)b|x))c', 'abc')) # Group used before defined. - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?(2)b|x))(c)') self.assertIsNone(re.match(r'(a)b(?<=(?(1)c|x))(c)', 'abc')) self.assertTrue(re.match(r'(a)b(?<=(?(1)b|x))(c)', 'abc')) # Group defined in the same lookbehind pattern - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)\2)(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?P.)(?P=a))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)\2)(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?P.)(?P=a))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') def test_ignore_case(self): self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") @@ -1318,8 +1321,8 @@ def test_sre_byte_literals(self): self.assertTrue(re.match((r"\x%02x" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"\x%02x0" % i).encode(), bytes([i])+b"0")) self.assertTrue(re.match((r"\x%02xz" % i).encode(), bytes([i])+b"z")) - self.assertRaises(re.error, re.compile, br"\u1234") - self.assertRaises(re.error, re.compile, br"\U00012345") + self.assertRaises(re.PatternError, re.compile, br"\u1234") + self.assertRaises(re.PatternError, re.compile, br"\U00012345") self.assertTrue(re.match(br"\0", b"\000")) self.assertTrue(re.match(br"\08", b"\0008")) self.assertTrue(re.match(br"\01", b"\001")) @@ -1341,8 +1344,8 @@ def test_sre_byte_class_literals(self): self.assertTrue(re.match((r"[\x%02x]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02x0]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02xz]" % i).encode(), bytes([i]))) - self.assertRaises(re.error, re.compile, br"[\u1234]") - self.assertRaises(re.error, re.compile, br"[\U00012345]") + self.assertRaises(re.PatternError, re.compile, br"[\u1234]") + self.assertRaises(re.PatternError, re.compile, br"[\U00012345]") self.checkPatternError(br"[\567]", r'octal escape value \567 outside of ' r'range 0-0o377', 1) @@ -1675,11 +1678,11 @@ def test_ascii_and_unicode_flag(self): self.assertIsNone(pat.match(b'\xe0')) # Incompatibilities self.assertRaises(ValueError, re.compile, br'\w', re.UNICODE) - self.assertRaises(re.error, re.compile, br'(?u)\w') + self.assertRaises(re.PatternError, re.compile, br'(?u)\w') self.assertRaises(ValueError, re.compile, r'\w', re.UNICODE | re.ASCII) self.assertRaises(ValueError, re.compile, r'(?u)\w', re.ASCII) self.assertRaises(ValueError, re.compile, r'(?a)\w', re.UNICODE) - self.assertRaises(re.error, re.compile, r'(?au)\w') + self.assertRaises(re.PatternError, re.compile, r'(?au)\w') def test_locale_flag(self): enc = locale.getpreferredencoding() @@ -1720,11 +1723,11 @@ def test_locale_flag(self): self.assertIsNone(pat.match(bletter)) # Incompatibilities self.assertRaises(ValueError, re.compile, '', re.LOCALE) - self.assertRaises(re.error, re.compile, '(?L)') + self.assertRaises(re.PatternError, re.compile, '(?L)') self.assertRaises(ValueError, re.compile, b'', re.LOCALE | re.ASCII) self.assertRaises(ValueError, re.compile, b'(?L)', re.ASCII) self.assertRaises(ValueError, re.compile, b'(?a)', re.LOCALE) - self.assertRaises(re.error, re.compile, b'(?aL)') + self.assertRaises(re.PatternError, re.compile, b'(?aL)') def test_scoped_flags(self): self.assertTrue(re.match(r'(?i:a)b', 'Ab')) @@ -2060,7 +2063,7 @@ def test_locale_compiled(self): self.assertIsNone(p4.match(b'\xc5\xc5')) def test_error(self): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile('(\u20ac))') err = cm.exception self.assertIsInstance(err.pattern, str) @@ -2072,14 +2075,14 @@ def test_error(self): self.assertIn(' at position 3', str(err)) self.assertNotIn(' at position 3', err.msg) # Bytes pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(b'(\xa4))') err = cm.exception self.assertIsInstance(err.pattern, bytes) self.assertEqual(err.pattern, b'(\xa4))') self.assertEqual(err.pos, 3) # Multiline pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(""" ( abc @@ -2820,7 +2823,7 @@ def test_re_tests(self): with self.subTest(pattern=pattern, string=s): if outcome == SYNTAX_ERROR: # Expected a syntax error - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): re.compile(pattern) continue diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 835280f2281cde..5e0e6f8dfac651 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -5,6 +5,7 @@ import os import sys import tempfile +import textwrap import unittest from test.support import verbose from test.support.import_helper import import_module @@ -18,7 +19,7 @@ if hasattr(readline, "_READLINE_LIBRARY_VERSION"): is_editline = ("EditLine wrapper" in readline._READLINE_LIBRARY_VERSION) else: - is_editline = (readline.__doc__ and "libedit" in readline.__doc__) + is_editline = readline.backend == "editline" def setUpModule(): @@ -144,6 +145,9 @@ def test_init(self): TERM='xterm-256color') self.assertEqual(stdout, b'') + def test_backend(self): + self.assertIn(readline.backend, ("readline", "editline")) + auto_history_script = """\ import readline readline.set_auto_history({}) @@ -163,6 +167,25 @@ def test_auto_history_disabled(self): # end, so don't expect it in the output. self.assertIn(b"History length: 0", output) + def test_set_complete_delims(self): + script = textwrap.dedent(""" + import readline + def complete(text, state): + if state == 0 and text == "$": + return "$complete" + return None + if readline.backend == "editline": + readline.parse_and_bind(r'bind "\\t" rl_complete') + else: + readline.parse_and_bind(r'"\\t": complete') + readline.set_completer_delims(" \\t\\n") + readline.set_completer(complete) + print(input()) + """) + + output = run_pty(script, input=b"$\t\n") + self.assertIn(b"$complete", output) + def test_nonascii(self): loc = locale.setlocale(locale.LC_CTYPE, None) if loc in ('C', 'POSIX'): @@ -178,7 +201,7 @@ def test_nonascii(self): script = r"""import readline -is_editline = readline.__doc__ and "libedit" in readline.__doc__ +is_editline = readline.backend == "editline" inserted = "[\xEFnserted]" macro = "|t\xEB[after]" set_pre_input_hook = getattr(readline, "set_pre_input_hook", None) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index d7b9f801092498..e828941f6c779d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2031,6 +2031,25 @@ def test_dev_mode(self): self.check_executed_tests(output, tests, stats=len(tests), parallel=True) + def test_unload_tests(self): + # Test that unloading test modules does not break tests + # that import from other tests. + # The test execution order matters for this test. + # Both test_regrtest_a and test_regrtest_c which are executed before + # and after test_regrtest_b import a submodule from the test_regrtest_b + # package and use it in testing. test_regrtest_b itself does not import + # that submodule. + # Previously test_regrtest_c failed because test_regrtest_b.util in + # sys.modules was left after test_regrtest_a (making the import + # statement no-op), but new test_regrtest_b without the util attribute + # was imported for test_regrtest_b. + testdir = os.path.join(os.path.dirname(__file__), + 'regrtestdata', 'import_from_tests') + tests = [f'test_regrtest_{name}' for name in ('a', 'b', 'c')] + args = ['-Wd', '-E', '-bb', '-m', 'test', '--testdir=%s' % testdir, *tests] + output = self.run_python(args) + self.check_executed_tests(output, tests, stats=3) + def check_add_python_opts(self, option): # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python code = textwrap.dedent(r""" diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 7533376e015e73..a28d1595f44533 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -161,10 +161,11 @@ def foo(x): output = kill_python(p) self.assertEqual(p.returncode, 0) - traceback_lines = output.splitlines()[-7:-1] + traceback_lines = output.splitlines()[-8:-1] expected_lines = [ ' File "", line 1, in ', ' foo(0)', + ' ~~~^^^', ' File "", line 2, in foo', ' 1 / x', ' ~~^~~', diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 7347fca71be2fe..273ce2cf5c7dd2 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -2,6 +2,7 @@ from unittest.mock import patch import builtins import rlcompleter +from test.support import MISSING_C_DOCSTRINGS class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ @@ -40,12 +41,12 @@ def test_global_matches(self): # test with a customized namespace self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe()']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) self.assertEqual(self.completer.global_matches('eg'), ['egg(']) # XXX: see issue5256 self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe()']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) def test_attr_matches(self): # test with builtins namespace diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index d231e66b7b889f..8edd75e9907ec0 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -208,8 +208,7 @@ def test_rmtree_fails_on_symlink_onerror(self): errors = [] def onerror(*args): errors.append(args) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(link, onerror=onerror) + shutil.rmtree(link, onerror=onerror) self.assertEqual(len(errors), 1) self.assertIs(errors[0][0], os.path.islink) self.assertEqual(errors[0][1], link) @@ -270,8 +269,7 @@ def test_rmtree_fails_on_junctions_onerror(self): errors = [] def onerror(*args): errors.append(args) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(link, onerror=onerror) + shutil.rmtree(link, onerror=onerror) self.assertEqual(len(errors), 1) self.assertIs(errors[0][0], os.path.islink) self.assertEqual(errors[0][1], link) @@ -319,7 +317,7 @@ def test_rmtree_works_on_junctions(self): self.assertTrue(os.path.exists(dir3)) self.assertTrue(os.path.exists(file1)) - def test_rmtree_errors_onerror(self): + def test_rmtree_errors(self): # filename is guaranteed not to exist filename = tempfile.mktemp(dir=self.mkdtemp()) self.assertRaises(FileNotFoundError, shutil.rmtree, filename) @@ -328,8 +326,8 @@ def test_rmtree_errors_onerror(self): # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") + write_file(filename, "") with self.assertRaises(NotADirectoryError) as cm: shutil.rmtree(filename) self.assertEqual(cm.exception.filename, filename) @@ -337,11 +335,23 @@ def test_rmtree_errors_onerror(self): # test that ignore_errors option is honored shutil.rmtree(filename, ignore_errors=True) self.assertTrue(os.path.exists(filename)) + + self.assertRaises(TypeError, shutil.rmtree, None) + self.assertRaises(TypeError, shutil.rmtree, None, ignore_errors=True) + exc = TypeError if shutil.rmtree.avoids_symlink_attacks else NotImplementedError + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid') + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid', ignore_errors=True) + + def test_rmtree_errors_onerror(self): + tmpdir = self.mkdtemp() + filename = os.path.join(tmpdir, "tstfile") + write_file(filename, "") errors = [] def onerror(*args): errors.append(args) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(filename, onerror=onerror) + shutil.rmtree(filename, onerror=onerror) self.assertEqual(len(errors), 2) self.assertIs(errors[0][0], os.scandir) self.assertEqual(errors[0][1], filename) @@ -353,23 +363,9 @@ def onerror(*args): self.assertEqual(errors[1][2][1].filename, filename) def test_rmtree_errors_onexc(self): - # filename is guaranteed not to exist - filename = tempfile.mktemp(dir=self.mkdtemp()) - self.assertRaises(FileNotFoundError, shutil.rmtree, filename) - # test that ignore_errors option is honored - shutil.rmtree(filename, ignore_errors=True) - - # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") - with self.assertRaises(NotADirectoryError) as cm: - shutil.rmtree(filename) - self.assertEqual(cm.exception.filename, filename) - self.assertTrue(os.path.exists(filename)) - # test that ignore_errors option is honored - shutil.rmtree(filename, ignore_errors=True) - self.assertTrue(os.path.exists(filename)) + write_file(filename, "") errors = [] def onexc(*args): errors.append(args) @@ -410,8 +406,7 @@ def test_on_error(self): self.addCleanup(os.chmod, self.child_file_path, old_child_file_mode) self.addCleanup(os.chmod, self.child_dir_path, old_child_dir_mode) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(TESTFN, onerror=self.check_args_to_onerror) + shutil.rmtree(TESTFN, onerror=self.check_args_to_onerror) # Test whether onerror has actually been called. self.assertEqual(self.errorState, 3, "Expected call to onerror function did not happen.") @@ -537,8 +532,7 @@ def onexc(*args): self.addCleanup(os.chmod, self.child_file_path, old_child_file_mode) self.addCleanup(os.chmod, self.child_dir_path, old_child_dir_mode) - with self.assertWarns(DeprecationWarning): - shutil.rmtree(TESTFN, onerror=onerror, onexc=onexc) + shutil.rmtree(TESTFN, onerror=onerror, onexc=onexc) self.assertTrue(onexc_called) self.assertFalse(onerror_called) @@ -582,6 +576,41 @@ def _raiser(*args, **kwargs): self.assertFalse(shutil._use_fd_functions) self.assertFalse(shutil.rmtree.avoids_symlink_attacks) + @unittest.skipUnless(shutil._use_fd_functions, "requires safe rmtree") + def test_rmtree_fails_on_close(self): + # Test that the error handler is called for failed os.close() and that + # os.close() is only called once for a file descriptor. + tmp = self.mkdtemp() + dir1 = os.path.join(tmp, 'dir1') + os.mkdir(dir1) + dir2 = os.path.join(dir1, 'dir2') + os.mkdir(dir2) + def close(fd): + orig_close(fd) + nonlocal close_count + close_count += 1 + raise OSError + + close_count = 0 + with support.swap_attr(os, 'close', close) as orig_close: + with self.assertRaises(OSError): + shutil.rmtree(dir1) + self.assertTrue(os.path.isdir(dir2)) + self.assertEqual(close_count, 2) + + close_count = 0 + errors = [] + def onexc(*args): + errors.append(args) + with support.swap_attr(os, 'close', close) as orig_close: + shutil.rmtree(dir1, onexc=onexc) + self.assertEqual(len(errors), 2) + self.assertIs(errors[0][0], close) + self.assertEqual(errors[0][1], dir2) + self.assertIs(errors[1][0], close) + self.assertEqual(errors[1][1], dir1) + self.assertEqual(close_count, 2) + @unittest.skipUnless(shutil._use_fd_functions, "dir_fd is not supported") def test_rmtree_with_dir_fd(self): tmp_dir = self.mkdtemp() @@ -638,6 +667,63 @@ def test_rmtree_on_junction(self): finally: shutil.rmtree(TESTFN, ignore_errors=True) + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_rmtree_deleted_race_condition(self): + # bpo-37260 + # + # Test that a file or a directory deleted after it is enumerated + # by scandir() but before unlink() or rmdr() is called doesn't + # generate any errors. + def _onexc(fn, path, exc): + assert fn in (os.rmdir, os.unlink) + if not isinstance(exc, PermissionError): + raise + # Make the parent and the children writeable. + for p, mode in zip(paths, old_modes): + os.chmod(p, mode) + # Remove other dirs except one. + keep = next(p for p in dirs if p != path) + for p in dirs: + if p != keep: + os.rmdir(p) + # Remove other files except one. + keep = next(p for p in files if p != path) + for p in files: + if p != keep: + os.unlink(p) + + os.mkdir(TESTFN) + paths = [TESTFN] + [os.path.join(TESTFN, f'child{i}') + for i in range(6)] + dirs = paths[1::2] + files = paths[2::2] + for path in dirs: + os.mkdir(path) + for path in files: + write_file(path, '') + + old_modes = [os.stat(path).st_mode for path in paths] + + # Make the parent and the children non-writeable. + new_mode = stat.S_IREAD|stat.S_IEXEC + for path in reversed(paths): + os.chmod(path, new_mode) + + try: + shutil.rmtree(TESTFN, onexc=_onexc) + except: + # Test failed, so cleanup artifacts. + for path, mode in zip(paths, old_modes): + try: + os.chmod(path, mode) + except OSError: + pass + shutil.rmtree(TESTFN) + raise + class TestCopyTree(BaseTest, unittest.TestCase): @@ -1015,19 +1101,18 @@ def test_copymode_follow_symlinks(self): shutil.copymode(src, dst) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) # On Windows, os.chmod does not follow symlinks (issue #15411) - if os.name != 'nt': - # follow src link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow dst link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow both links - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow src link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow dst link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow both links + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod') @os_helper.skip_unless_symlink @@ -1046,10 +1131,11 @@ def test_copymode_symlink_to_symlink(self): os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) # link to link os.lchmod(dst_link, stat.S_IRWXO) + old_mode = os.stat(dst).st_mode shutil.copymode(src_link, dst_link, follow_symlinks=False) self.assertEqual(os.lstat(src_link).st_mode, os.lstat(dst_link).st_mode) - self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + self.assertEqual(os.stat(dst).st_mode, old_mode) # src link - use chmod os.lchmod(dst_link, stat.S_IRWXO) shutil.copymode(src_link, dst, follow_symlinks=False) @@ -1584,6 +1670,17 @@ def test_tarfile_vs_tar(self): # now create another tarball using `tar` tarball2 = os.path.join(root_dir, 'archive2.tar') tar_cmd = ['tar', '-cf', 'archive2.tar', base_dir] + if sys.platform == 'darwin': + # macOS tar can include extended attributes, + # ACLs and other mac specific metadata into the + # archive (an recentish version of the OS). + # + # This feature can be disabled with the + # '--no-mac-metadata' option on macOS 11 or + # later. + import platform + if int(platform.mac_ver()[0].split('.')[0]) >= 11: + tar_cmd.insert(1, '--no-mac-metadata') subprocess.check_call(tar_cmd, cwd=root_dir, stdout=subprocess.DEVNULL) @@ -2591,6 +2688,35 @@ def test_move_dir_caseinsensitive(self): finally: os.rmdir(dst_dir) + # bpo-26791: Check that a symlink to a directory can + # be moved into that directory. + @mock_rename + def _test_move_symlink_to_dir_into_dir(self, dst): + src = os.path.join(self.src_dir, 'linktodir') + dst_link = os.path.join(self.dst_dir, 'linktodir') + os.symlink(self.dst_dir, src, target_is_directory=True) + shutil.move(src, dst) + self.assertTrue(os.path.islink(dst_link)) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertFalse(os.path.exists(src)) + + # Repeat the move operation with the destination + # symlink already in place (should raise shutil.Error). + os.symlink(self.dst_dir, src, target_is_directory=True) + with self.assertRaises(shutil.Error): + shutil.move(src, dst) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertTrue(os.path.exists(src)) + + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_dir(self): + self._test_move_symlink_to_dir_into_dir(self.dst_dir) + + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_symlink_to_dir(self): + dst = os.path.join(self.src_dir, 'otherlinktodir') + os.symlink(self.dst_dir, dst, target_is_directory=True) + self._test_move_symlink_to_dir_into_dir(dst) @os_helper.skip_unless_dac_override @unittest.skipUnless(hasattr(os, 'lchflags') diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index f2ae28c38dd72d..637a0ca3b36972 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1,5 +1,6 @@ import enum import errno +import functools import inspect import os import random @@ -76,6 +77,9 @@ class PosixTests(unittest.TestCase): def trivial_signal_handler(self, *args): pass + def create_handler_with_partial(self, argument): + return functools.partial(self.trivial_signal_handler, argument) + def test_out_of_range_signal_number_raises_error(self): self.assertRaises(ValueError, signal.getsignal, 4242) @@ -96,6 +100,28 @@ def test_getsignal(self): signal.signal(signal.SIGHUP, hup) self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + def test_no_repr_is_called_on_signal_handler(self): + # See https://github.com/python/cpython/issues/112559. + + class MyArgument: + def __init__(self): + self.repr_count = 0 + + def __repr__(self): + self.repr_count += 1 + return super().__repr__() + + argument = MyArgument() + self.assertEqual(0, argument.repr_count) + + handler = self.create_handler_with_partial(argument) + hup = signal.signal(signal.SIGHUP, handler) + self.assertIsInstance(hup, signal.Handlers) + self.assertEqual(signal.getsignal(signal.SIGHUP), handler) + signal.signal(signal.SIGHUP, hup) + self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + self.assertEqual(0, argument.repr_count) + def test_strsignal(self): self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) @@ -1318,6 +1344,7 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") + @unittest.skipIf(sys.platform == "darwin", "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") @threading_helper.requires_working_threading() diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 33d0975bda8eaa..9f199d9069d207 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -641,10 +641,24 @@ def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): sys_path.append(abs_path) return sys_path + def _get_pth_lines(self, libpath: str, *, import_site: bool): + pth_lines = ['fake-path-name'] + # include 200 lines of `libpath` in _pth lines (or fewer + # if the `libpath` is long enough to get close to 32KB + # see https://github.com/python/cpython/issues/113628) + encoded_libpath_length = len(libpath.encode("utf-8")) + repetitions = min(200, 30000 // encoded_libpath_length) + if repetitions <= 2: + self.skipTest( + f"Python stdlib path is too long ({encoded_libpath_length:,} bytes)") + pth_lines.extend(libpath for _ in range(repetitions)) + pth_lines.extend(['', '# comment']) + if import_site: + pth_lines.append('import site') + return pth_lines + @support.requires_subprocess() def test_underpth_basic(self): - libpath = test.support.STDLIB_DIR - exe_prefix = os.path.dirname(sys.executable) pth_lines = ['#.', '# ..', *sys.path, '.', '..'] exe_file = self._create_underpth_exe(pth_lines) sys_path = self._calc_sys_path_for_underpth_nosite( @@ -666,12 +680,7 @@ def test_underpth_basic(self): def test_underpth_nosite_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - pth_lines = [ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - ] + pth_lines = self._get_pth_lines(libpath, import_site=False) exe_file = self._create_underpth_exe(pth_lines) sys_path = self._calc_sys_path_for_underpth_nosite( os.path.dirname(exe_file), @@ -695,13 +704,8 @@ def test_underpth_nosite_file(self): def test_underpth_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - exe_file = self._create_underpth_exe([ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - 'import site' - ]) + exe_file = self._create_underpth_exe( + self._get_pth_lines(libpath, import_site=True)) sys_prefix = os.path.dirname(exe_file) env = os.environ.copy() env['PYTHONPATH'] = 'from-env' @@ -720,13 +724,8 @@ def test_underpth_file(self): def test_underpth_dll_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - exe_file = self._create_underpth_exe([ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - 'import site' - ], exe_pth=False) + exe_file = self._create_underpth_exe( + self._get_pth_lines(libpath, import_site=True), exe_pth=False) sys_prefix = os.path.dirname(exe_file) env = os.environ.copy() env['PYTHONPATH'] = 'from-env' diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 86701caf05399e..4eb5af99d6674c 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1082,7 +1082,20 @@ def testInterfaceNameIndex(self): 'socket.if_indextoname() not available.') def testInvalidInterfaceIndexToName(self): self.assertRaises(OSError, socket.if_indextoname, 0) + self.assertRaises(OverflowError, socket.if_indextoname, -1) + self.assertRaises(OverflowError, socket.if_indextoname, 2**1000) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + if hasattr(socket, 'if_nameindex'): + indices = dict(socket.if_nameindex()) + for index in indices: + index2 = index + 2**32 + if index2 not in indices: + with self.assertRaises((OverflowError, OSError)): + socket.if_indextoname(index2) + for index in 2**32-1, 2**64-1: + if index not in indices: + with self.assertRaises((OverflowError, OSError)): + socket.if_indextoname(index) @unittest.skipUnless(hasattr(socket, 'if_nametoindex'), 'socket.if_nametoindex() not available.') diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 9ade595ef8ae7e..3fdfa2960503b8 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -10,6 +10,7 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import asyncore +import array import re import socket import select @@ -3517,6 +3518,27 @@ def test_recv_zero(self): self.assertEqual(s.recv(0), b"") self.assertEqual(s.recv_into(bytearray()), 0) + def test_recv_into_buffer_protocol_len(self): + server = ThreadedEchoServer(CERTFILE) + self.enterContext(server) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = test_wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + s.send(b"data") + buf = array.array('I', [0, 0]) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf)[:4], b"data") + + class B(bytearray): + def __len__(self): + 1/0 + s.send(b"data") + buf = B(6) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf), b"data\0\0") + def test_nonblocking_send(self): server = ThreadedEchoServer(CERTFILE, certreqs=ssl.CERT_NONE, @@ -4237,6 +4259,7 @@ def test_session_handling(self): 'Session refers to a different SSLContext.') @requires_tls_version('TLSv1_2') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') def test_psk(self): psk = bytes.fromhex('deadbeef') @@ -4304,6 +4327,7 @@ def server_callback(identity): s.connect((HOST, server.port)) @requires_tls_version('TLSv1_3') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') def test_psk_tls1_3(self): psk = bytes.fromhex('deadbeef') identity_hint = 'identity-hint' diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index 0eced2fcf98376..a0d0f61e5a192c 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -148,12 +148,19 @@ def test_mode(self): self.assertEqual(modestr, '-r--r--r--') self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) else: + os.chmod(TESTFN, 0o500) + st_mode, modestr = self.get_mode() + self.assertEqual(modestr[:3], '-r-') + self.assertS_IS("REG", st_mode) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) + os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() self.assertEqual(modestr[:3], '-rw') self.assertS_IS("REG", st_mode) self.assertEqual(self.statmod.S_IFMT(st_mode), self.statmod.S_IFREG) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o666) @os_helper.skip_unless_working_chmod def test_directory(self): diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index b24fc3c3d077fe..bf2c254c9ee7d9 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2302,10 +2302,12 @@ def test_error_cases(self): StatisticsError = statistics.StatisticsError with self.assertRaises(StatisticsError): geometric_mean([]) # empty input - with self.assertRaises(StatisticsError): - geometric_mean([3.5, 0.0, 5.25]) # zero input with self.assertRaises(StatisticsError): geometric_mean([3.5, -4.0, 5.25]) # negative input + with self.assertRaises(StatisticsError): + geometric_mean([0.0, -4.0, 5.25]) # negative input with zero + with self.assertRaises(StatisticsError): + geometric_mean([3.5, -math.inf, 5.25]) # negative infinity with self.assertRaises(StatisticsError): geometric_mean(iter([])) # empty iterator with self.assertRaises(TypeError): @@ -2328,6 +2330,12 @@ def test_special_values(self): with self.assertRaises(ValueError): geometric_mean([Inf, -Inf]) + # Cases with zero + self.assertEqual(geometric_mean([3, 0.0, 5]), 0.0) # Any zero gives a zero + self.assertEqual(geometric_mean([3, -0.0, 5]), 0.0) # Negative zero allowed + self.assertTrue(math.isnan(geometric_mean([0, NaN]))) # NaN beats zero + self.assertTrue(math.isnan(geometric_mean([0, Inf]))) # Because 0.0 * Inf -> NaN + def test_mixed_int_and_float(self): # Regression test for b.p.o. issue #28327 geometric_mean = statistics.geometric_mean diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 814ef111c5bec8..b4927113db44e3 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -55,6 +55,21 @@ def duplicate_string(text): class StrSubclass(str): pass +class OtherStrSubclass(str): + pass + +class WithStr: + def __init__(self, value): + self.value = value + def __str__(self): + return self.value + +class WithRepr: + def __init__(self, value): + self.value = value + def __repr__(self): + return self.value + class StrTest(string_tests.StringLikeTest, string_tests.MixinStrUnicodeTest, unittest.TestCase): @@ -83,6 +98,10 @@ def __repr__(self): self.assertEqual(realresult, result) self.assertTrue(object is not realresult) + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + def test_literals(self): self.assertEqual('\xff', '\u00ff') self.assertEqual('\uffff', '\U0000ffff') @@ -127,10 +146,13 @@ def test_ascii(self): self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096), ascii("\U00010000" * 39 + "\uffff" * 4096)) - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, ascii, WrongRepr()) + self.assertTypedEqual(ascii('\U0001f40d'), r"'\U0001f40d'") + self.assertTypedEqual(ascii(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(ascii(WithRepr('')), '') + self.assertTypedEqual(ascii(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') + self.assertTypedEqual(ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') + self.assertRaises(TypeError, ascii, WithRepr(b'byte-repr')) def test_repr(self): # Test basic sanity of repr() @@ -168,10 +190,13 @@ def test_repr(self): self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096), repr("\U00010000" * 39 + "\uffff" * 4096)) - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, repr, WrongRepr()) + self.assertTypedEqual(repr('\U0001f40d'), "'\U0001f40d'") + self.assertTypedEqual(repr(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(repr(WithRepr('')), '') + self.assertTypedEqual(repr(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') + self.assertTypedEqual(repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) + self.assertRaises(TypeError, repr, WithRepr(b'byte-repr')) def test_iterators(self): # Make sure unicode objects have an __iter__ method @@ -2367,28 +2392,37 @@ def test_ucs4(self): def test_conversion(self): # Make sure __str__() works properly - class ObjectToStr: - def __str__(self): - return "foo" - - class StrSubclassToStr(str): - def __str__(self): - return "foo" - - class StrSubclassToStrSubclass(str): - def __new__(cls, content=""): - return str.__new__(cls, 2*content) - def __str__(self): + class StrWithStr(str): + def __new__(cls, value): + self = str.__new__(cls, "") + self.value = value return self + def __str__(self): + return self.value - self.assertEqual(str(ObjectToStr()), "foo") - self.assertEqual(str(StrSubclassToStr("bar")), "foo") - s = str(StrSubclassToStrSubclass("foo")) - self.assertEqual(s, "foofoo") - self.assertIs(type(s), StrSubclassToStrSubclass) - s = StrSubclass(StrSubclassToStrSubclass("foo")) - self.assertEqual(s, "foofoo") - self.assertIs(type(s), StrSubclass) + self.assertTypedEqual(str(WithStr('abc')), 'abc') + self.assertTypedEqual(str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr('abc')), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr(StrSubclass('abc'))), + StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr(OtherStrSubclass('abc'))), + StrSubclass('abc')) + + self.assertTypedEqual(str(StrWithStr('abc')), 'abc') + self.assertTypedEqual(str(StrWithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr('abc')), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr(StrSubclass('abc'))), + StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr(OtherStrSubclass('abc'))), + StrSubclass('abc')) + + self.assertTypedEqual(str(WithRepr('')), '') + self.assertTypedEqual(str(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(StrSubclass(WithRepr('')), StrSubclass('')) + self.assertTypedEqual(StrSubclass(WithRepr(StrSubclass(''))), + StrSubclass('')) + self.assertTypedEqual(StrSubclass(WithRepr(OtherStrSubclass(''))), + StrSubclass('')) def test_unicode_repr(self): class s1: diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 810c5a36e02f41..05c8afc907ad3c 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -224,35 +224,55 @@ def test_ValueError(self): else: self.fail("'%s' did not raise ValueError" % bad_format) + msg_week_no_year_or_weekday = r"ISO week directive '%V' must be used with " \ + r"the ISO year directive '%G' and a weekday directive " \ + r"\('%A', '%a', '%w', or '%u'\)." + msg_week_not_compatible = r"ISO week directive '%V' is incompatible with " \ + r"the year directive '%Y'. Use the ISO year '%G' instead." + msg_julian_not_compatible = r"Day of the year directive '%j' is not " \ + r"compatible with ISO year directive '%G'. Use '%Y' instead." + msg_year_no_week_or_weekday = r"ISO year directive '%G' must be used with " \ + r"the ISO week directive '%V' and a weekday directive " \ + r"\('%A', '%a', '%w', or '%u'\)." + + locale_time = _strptime.LocaleTime() + # Ambiguous or incomplete cases using ISO year/week/weekday directives - # 1. ISO week (%V) is specified, but the year is specified with %Y - # instead of %G - with self.assertRaises(ValueError): - _strptime._strptime("1999 50", "%Y %V") - # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not - with self.assertRaises(ValueError): - _strptime._strptime("1999 51", "%G %V") - # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not - for w in ('A', 'a', 'w', 'u'): - with self.assertRaises(ValueError): - _strptime._strptime("1999 51","%G %{}".format(w)) - # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) - with self.assertRaises(ValueError): - _strptime._strptime("2015", "%G") - # 5. Julian/ordinal day (%j) is specified with %G, but not %Y - with self.assertRaises(ValueError): - _strptime._strptime("1999 256", "%G %j") - # 6. Invalid ISO weeks - invalid_iso_weeks = [ - "2019-00-1", - "2019-54-1", - "2021-53-1", + subtests = [ + # 1. ISO week (%V) is specified, but the year is specified with %Y + # instead of %G + ("1999 50", "%Y %V", msg_week_no_year_or_weekday), + ("1999 50 5", "%Y %V %u", msg_week_not_compatible), + # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not + ("1999 51", "%G %V", msg_year_no_week_or_weekday), + # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not + ("1999 {}".format(locale_time.f_weekday[5]), "%G %A", + msg_year_no_week_or_weekday), + ("1999 {}".format(locale_time.a_weekday[5]), "%G %a", + msg_year_no_week_or_weekday), + ("1999 5", "%G %w", msg_year_no_week_or_weekday), + ("1999 5", "%G %u", msg_year_no_week_or_weekday), + # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) + ("2015", "%G", msg_year_no_week_or_weekday), + # 5. Julian/ordinal day (%j) is specified with %G, but not %Y + ("1999 256", "%G %j", msg_julian_not_compatible), + ("1999 50 5 256", "%G %V %u %j", msg_julian_not_compatible), + # ISO week specified alone + ("50", "%V", msg_week_no_year_or_weekday), + # ISO year is unspecified, falling back to year + ("50 5", "%V %u", msg_week_no_year_or_weekday), + # 6. Invalid ISO weeks + ("2019-00-1", "%G-%V-%u", + "time data '2019-00-1' does not match format '%G-%V-%u'"), + ("2019-54-1", "%G-%V-%u", + "time data '2019-54-1' does not match format '%G-%V-%u'"), + ("2021-53-1", "%G-%V-%u", "Invalid week: 53"), ] - for invalid_iso_dtstr in invalid_iso_weeks: - with self.subTest(invalid_iso_dtstr): - with self.assertRaises(ValueError): - _strptime._strptime(invalid_iso_dtstr, "%G-%V-%u") + for (data_string, format, message) in subtests: + with self.subTest(data_string=data_string, format=format): + with self.assertRaisesRegex(ValueError, message): + _strptime._strptime(data_string, format) def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index fe1a3675fced65..6d3228bf92f8ca 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1561,21 +1561,6 @@ def test_class_getitems(self): self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) - @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), - "vfork() not enabled by configure.") - @mock.patch("subprocess._fork_exec") - def test__use_vfork(self, mock_fork_exec): - self.assertTrue(subprocess._USE_VFORK) # The default value regardless. - mock_fork_exec.side_effect = RuntimeError("just testing args") - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - mock_fork_exec.assert_called_once() - self.assertTrue(mock_fork_exec.call_args.args[-1]) - with mock.patch.object(subprocess, '_USE_VFORK', False): - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) - class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): @@ -2066,8 +2051,14 @@ def test_group_error(self): def test_extra_groups(self): gid = os.getegid() group_list = [65534 if gid != 65534 else 65533] + self._test_extra_groups_impl(gid=gid, group_list=group_list) + + @unittest.skipUnless(hasattr(os, 'setgroups'), 'no setgroups() on platform') + def test_extra_groups_empty_list(self): + self._test_extra_groups_impl(gid=os.getegid(), group_list=[]) + + def _test_extra_groups_impl(self, *, gid, group_list): name_group = _get_test_grp_name() - perm_error = False if grp is not None: group_list.append(name_group) @@ -2077,11 +2068,8 @@ def test_extra_groups(self): [sys.executable, "-c", "import os, sys, json; json.dump(os.getgroups(), sys.stdout)"], extra_groups=group_list) - except OSError as ex: - if ex.errno != errno.EPERM: - raise - perm_error = True - + except PermissionError: + self.skipTest("setgroup() EPERM; this test may require root.") else: parent_groups = os.getgroups() child_groups = json.loads(output) @@ -2092,12 +2080,15 @@ def test_extra_groups(self): else: desired_gids = group_list - if perm_error: - self.assertEqual(set(child_groups), set(parent_groups)) - else: - self.assertEqual(set(desired_gids), set(child_groups)) + self.assertEqual(set(desired_gids), set(child_groups)) - # make sure we bomb on negative values + if grp is None: + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, + extra_groups=[name_group]) + + # No skip necessary, this test won't make it to a setgroup() call. + def test_extra_groups_invalid_gid_t_values(self): with self.assertRaises(ValueError): subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[-1]) @@ -2106,16 +2097,6 @@ def test_extra_groups(self): cwd=os.curdir, env=os.environ, extra_groups=[2**64]) - if grp is None: - with self.assertRaises(ValueError): - subprocess.check_call(ZERO_RETURN_CMD, - extra_groups=[name_group]) - - @unittest.skipIf(hasattr(os, 'setgroups'), 'setgroups() available on platform') - def test_extra_groups_error(self): - with self.assertRaises(ValueError): - subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[]) - @unittest.skipIf(mswindows or not hasattr(os, 'umask'), 'POSIX umask() is not available.') def test_umask(self): @@ -3364,6 +3345,94 @@ def exit_handler(): self.assertEqual(out, b'') self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) + @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), + "vfork() not enabled by configure.") + @mock.patch("subprocess._fork_exec") + @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) + def test__use_vfork(self, mock_fork_exec): + self.assertTrue(subprocess._USE_VFORK) # The default value regardless. + mock_fork_exec.side_effect = RuntimeError("just testing args") + with self.assertRaises(RuntimeError): + subprocess.run([sys.executable, "-c", "pass"]) + mock_fork_exec.assert_called_once() + # NOTE: These assertions are *ugly* as they require the last arg + # to remain the have_vfork boolean. We really need to refactor away + # from the giant "wall of args" internal C extension API. + self.assertTrue(mock_fork_exec.call_args.args[-1]) + with mock.patch.object(subprocess, '_USE_VFORK', False): + with self.assertRaises(RuntimeError): + subprocess.run([sys.executable, "-c", "pass"]) + self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) + + @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), + "vfork() not enabled by configure.") + @unittest.skipIf(sys.platform != "linux", "Linux only, requires strace.") + @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) + def test_vfork_used_when_expected(self): + # This is a performance regression test to ensure we default to using + # vfork() when possible. + # Technically this test could pass when posix_spawn is used as well + # because libc tends to implement that internally using vfork. But + # that'd just be testing a libc+kernel implementation detail. + strace_binary = "/usr/bin/strace" + # The only system calls we are interested in. + strace_filter = "--trace=clone,clone2,clone3,fork,vfork,exit,exit_group" + true_binary = "/bin/true" + strace_command = [strace_binary, strace_filter] + + try: + does_strace_work_process = subprocess.run( + strace_command + [true_binary], + stderr=subprocess.PIPE, + stdout=subprocess.DEVNULL, + ) + rc = does_strace_work_process.returncode + stderr = does_strace_work_process.stderr + except OSError: + rc = -1 + stderr = "" + if rc or (b"+++ exited with 0 +++" not in stderr): + self.skipTest("strace not found or not working as expected.") + + with self.subTest(name="default_is_vfork"): + vfork_result = assert_python_ok( + "-c", + textwrap.dedent(f"""\ + import subprocess + subprocess.check_call([{true_binary!r}])"""), + __run_using_command=strace_command, + ) + # Match both vfork() and clone(..., flags=...|CLONE_VFORK|...) + self.assertRegex(vfork_result.err, br"(?i)vfork") + # Do NOT check that fork() or other clones did not happen. + # If the OS denys the vfork it'll fallback to plain fork(). + + # Test that each individual thing that would disable the use of vfork + # actually disables it. + for sub_name, preamble, sp_kwarg, expect_permission_error in ( + ("!use_vfork", "subprocess._USE_VFORK = False", "", False), + ("preexec", "", "preexec_fn=lambda: None", False), + ("setgid", "", f"group={os.getgid()}", True), + ("setuid", "", f"user={os.getuid()}", True), + ("setgroups", "", "extra_groups=[]", True), + ): + with self.subTest(name=sub_name): + non_vfork_result = assert_python_ok( + "-c", + textwrap.dedent(f"""\ + import subprocess + {preamble} + try: + subprocess.check_call( + [{true_binary!r}], **dict({sp_kwarg})) + except PermissionError: + if not {expect_permission_error}: + raise"""), + __run_using_command=strace_command, + ) + # Ensure neither vfork() or clone(..., flags=...|CLONE_VFORK|...). + self.assertNotRegex(non_vfork_result.err, br"(?i)vfork") + @unittest.skipUnless(mswindows, "Windows specific tests") class Win32ProcessTestCase(BaseTestCase): diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 43162c540b55ae..f8e968b9b56f82 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -396,6 +396,33 @@ def method(self): with self.assertRaisesRegex(TypeError, "argument 1 must be a type"): C().method() + def test_supercheck_fail(self): + class C: + def method(self, type_, obj): + return super(type_, obj).method() + + c = C() + err_msg = ( + r"super\(type, obj\): obj \({} {}\) is not " + r"an instance or subtype of type \({}\)." + ) + + cases = ( + (int, c, int.__name__, C.__name__, "instance of"), + # obj is instance of type + (C, list(), C.__name__, list.__name__, "instance of"), + # obj is type itself + (C, list, C.__name__, list.__name__, "type"), + ) + + for case in cases: + with self.subTest(case=case): + type_, obj, type_str, obj_str, instance_or_type = case + regex = err_msg.format(instance_or_type, obj_str, type_str) + + with self.assertRaisesRegex(TypeError, regex): + c.method(type_, obj) + def test_super___class__(self): class C: def method(self): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index c34b0e5e015702..d160cbf0645b47 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -630,7 +630,7 @@ def recursive_function(depth): if depth: recursive_function(depth - 1) - for max_depth in (5, 25, 250): + for max_depth in (5, 25, 250, 2500): with support.infinite_recursion(max_depth): available = support.get_recursion_available() diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 987e9e32afc325..92b78a8086a83d 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -337,7 +337,6 @@ def test_stdin(self): symtable.main(['-']) self.assertEqual(stdout.getvalue(), out) lines = out.splitlines() - print(out) self.assertIn("symbol table for module from file '':", lines) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index f6fa6495508d2c..83cbf5ec865dbb 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2017,6 +2017,7 @@ def f(x: *b) import re import doctest +import textwrap import unittest from test import support @@ -2279,6 +2280,31 @@ def test_nested_named_except_blocks(self): code += f"{' '*4*12}pass" self._check_error(code, "too many statically nested blocks") + @support.cpython_only + def test_with_statement_many_context_managers(self): + # See gh-113297 + + def get_code(n): + code = textwrap.dedent(""" + def bug(): + with ( + a + """) + for i in range(n): + code += f" as a{i}, a\n" + code += "): yield a" + return code + + CO_MAXBLOCKS = 20 # static nesting limit of the compiler + + for n in range(CO_MAXBLOCKS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "", "exec") + + for n in range(CO_MAXBLOCKS, CO_MAXBLOCKS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + def test_barry_as_flufl_with_syntax_errors(self): # The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if # is reading the wrong token in the presence of syntax errors later @@ -2334,6 +2360,12 @@ def test_error_parenthesis(self): """ self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + self._check_error("match y:\n case e(e=v,v,", " was never closed") + + # Examples with dencodings + s = b'# coding=latin\n(aaaaaaaaaaaaaaaaa\naaaaaaaaaaa\xb5' + self._check_error(s, r"'\(' was never closed") + def test_error_string_literal(self): self._check_error("'blech", r"unterminated string literal \(.*\)$") @@ -2349,6 +2381,7 @@ def test_error_string_literal(self): def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") + self._check_error(b"with(0,,):\n\x01", "invalid non-printable character") def test_match_call_does_not_raise_syntax_error(self): code = """ diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 7f49fb004272bb..6c87dfabad9f0f 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -691,11 +691,23 @@ def test_43581(self): self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding) def test_intern(self): + has_is_interned = (test.support.check_impl_detail(cpython=True) + or hasattr(sys, '_is_interned')) self.assertRaises(TypeError, sys.intern) + self.assertRaises(TypeError, sys.intern, b'abc') + if has_is_interned: + self.assertRaises(TypeError, sys._is_interned) + self.assertRaises(TypeError, sys._is_interned, b'abc') s = "never interned before" + str(random.randrange(0, 10**9)) self.assertTrue(sys.intern(s) is s) + if has_is_interned: + self.assertIs(sys._is_interned(s), True) s2 = s.swapcase().swapcase() + if has_is_interned: + self.assertIs(sys._is_interned(s2), False) self.assertTrue(sys.intern(s2) is s) + if has_is_interned: + self.assertIs(sys._is_interned(s2), False) # Subclasses of string can't be interned, because they # provide too much opportunity for insane things to happen. @@ -707,6 +719,8 @@ def __hash__(self): return 123 self.assertRaises(TypeError, sys.intern, S("abc")) + if has_is_interned: + self.assertIs(sys._is_interned(S("abc")), False) @requires_subinterpreters def test_subinterp_intern_dynamically_allocated(self): @@ -715,7 +729,7 @@ def test_subinterp_intern_dynamically_allocated(self): self.assertIs(t, s) interp = interpreters.create() - interp.run(textwrap.dedent(f''' + interp.exec_sync(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) != {id(s)}, (id(t), {id(s)}) @@ -730,7 +744,7 @@ def test_subinterp_intern_statically_allocated(self): t = sys.intern(s) interp = interpreters.create() - interp.run(textwrap.dedent(f''' + interp.exec_sync(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) == {id(t)}, (id(t), {id(t)}) @@ -1115,8 +1129,10 @@ def check(tracebacklimit, expected): b'Traceback (most recent call last):', b' File "", line 8, in ', b' f2()', + b' ~~^^', b' File "", line 6, in f2', b' f1()', + b' ~~^^', b' File "", line 4, in f1', b' 1 / 0', b' ~~^~~', @@ -1124,8 +1140,8 @@ def check(tracebacklimit, expected): ] check(10, traceback) check(3, traceback) - check(2, traceback[:1] + traceback[3:]) - check(1, traceback[:1] + traceback[5:]) + check(2, traceback[:1] + traceback[4:]) + check(1, traceback[:1] + traceback[7:]) check(0, [traceback[-1]]) check(-1, [traceback[-1]]) check(1<<1000, traceback) @@ -1208,9 +1224,7 @@ def test_pystats(self): @test.support.cpython_only @unittest.skipUnless(hasattr(sys, 'abiflags'), 'need sys.abiflags') def test_disable_gil_abi(self): - abi_threaded = 't' in sys.abiflags - py_gil_disabled = (sysconfig.get_config_var('Py_GIL_DISABLED') == 1) - self.assertEqual(py_gil_disabled, abi_threaded) + self.assertEqual('t' in sys.abiflags, support.Py_GIL_DISABLED) @test.support.cpython_only diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 2a6813f00bccc6..be609a0abd29c8 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -43,6 +43,7 @@ def setUp(self): self.name = os.name self.platform = sys.platform self.version = sys.version + self._framework = sys._framework self.sep = os.sep self.join = os.path.join self.isabs = os.path.isabs @@ -66,6 +67,7 @@ def tearDown(self): os.name = self.name sys.platform = self.platform sys.version = self.version + sys._framework = self._framework os.sep = self.sep os.path.join = self.join os.path.isabs = self.isabs @@ -139,7 +141,7 @@ def test_get_preferred_schemes(self): # Mac, framework build. os.name = 'posix' sys.platform = 'darwin' - sys._framework = True + sys._framework = "MyPython" self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) @@ -413,7 +415,10 @@ def test_library(self): else: self.assertTrue(library.startswith(f'libpython{major}.{minor}')) self.assertTrue(library.endswith('.a')) - self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + if sys.platform == 'darwin' and sys._framework: + self.skipTest('gh-110824: skip LDLIBRARY test for framework build') + else: + self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() @@ -472,11 +477,15 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - # /PC/pyconfig.h always exists even if unused on POSIX. - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + # /PC/pyconfig.h.in always exists even if unused + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) + if os.name == 'nt': + # /pyconfig.h exists on Windows in a build tree + pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 761560bfbf8b53..da5009126b3815 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -323,11 +323,23 @@ def test_list_verbose(self): # accessories if verbose flag is being used # ... # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/conttype - # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/regtype + # -rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/regtype + # drwxr-xr-x tarfile/tarfile 0 2003-01-05 15:19:43 ustar/dirtype/ # ... - self.assertRegex(out, (br'\?rw-r--r-- tarfile/tarfile\s+7011 ' - br'\d{4}-\d\d-\d\d\s+\d\d:\d\d:\d\d ' - br'ustar/\w+type ?\r?\n') * 2) + # + # Array of values to modify the regex below: + # ((file_type, file_permissions, file_length), ...) + type_perm_lengths = ( + (br'\?', b'rw-r--r--', b'7011'), (b'-', b'rw-r--r--', b'7011'), + (b'd', b'rwxr-xr-x', b'0'), (b'd', b'rwxr-xr-x', b'255'), + (br'\?', b'rw-r--r--', b'0'), (b'l', b'rwxrwxrwx', b'0'), + (b'b', b'rw-rw----', b'3,0'), (b'c', b'rw-rw-rw-', b'1,3'), + (b'p', b'rw-r--r--', b'0')) + self.assertRegex(out, b''.join( + [(tp + (br'%s tarfile/tarfile\s+%s ' % (perm, ln) + + br'\d{4}-\d\d-\d\d\s+\d\d:\d\d:\d\d ' + br'ustar/\w+type[/>\sa-z-]*\n')) for tp, perm, ln + in type_perm_lengths])) # Make sure it prints the source of link with verbose flag self.assertIn(b'ustar/symtype -> regtype', out) self.assertIn(b'./ustar/linktest2/symtype -> ../linktest1/regtype', out) @@ -3452,7 +3464,7 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None, path = pathlib.Path(os.path.normpath(self.destdir / name)) self.assertIn(path, self.expected_paths) self.expected_paths.remove(path) - if mode is not None and os_helper.can_chmod(): + if mode is not None and os_helper.can_chmod() and os.name != 'nt': got = stat.filemode(stat.S_IMODE(path.stat().st_mode)) self.assertEqual(got, mode) if type is None and isinstance(name, str) and name.endswith('/'): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 1673507e2f7c91..b64b6a4f2baeb5 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1641,6 +1641,28 @@ def test_explicit_cleanup_ignore_errors(self): temp_path.exists(), f"TemporaryDirectory {temp_path!s} exists after cleanup") + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_explicit_cleanup_correct_error(self): + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create(dir=working_dir) + with open(os.path.join(temp_dir.name, "example.txt"), 'wb'): + # Previously raised NotADirectoryError on some OSes + # (e.g. Windows). See bpo-43153. + with self.assertRaises(PermissionError): + temp_dir.cleanup() + + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_cleanup_with_used_directory(self): + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create(dir=working_dir) + subdir = os.path.join(temp_dir.name, "subdir") + os.mkdir(subdir) + with os_helper.change_cwd(subdir): + # Previously raised RecursionError on some OSes + # (e.g. Windows). See bpo-35144. + with self.assertRaises(PermissionError): + temp_dir.cleanup() + @os_helper.skip_unless_symlink def test_cleanup_with_symlink_to_a_directory(self): # cleanup() should not follow symlinks to directories (issue #12464) @@ -1662,6 +1684,103 @@ def test_cleanup_with_symlink_to_a_directory(self): "were deleted") d2.cleanup() + @os_helper.skip_unless_symlink + def test_cleanup_with_symlink_modes(self): + # cleanup() should not follow symlinks when fixing mode bits (#91133) + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + for mode in range(8): + mode <<= 6 + with self.subTest(mode=format(mode, '03o')): + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chmod(symlink, mode, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chmod(symlink, mode) + except FileNotFoundError: + pass + os.chmod(d1.name, mode) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chmod(file1, mode) + old_mode = os.stat(file1).st_mode + test(file1, target_is_directory=False) + new_mode = os.stat(file1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + with self.subTest('existing dir'): + os.chmod(dir1, mode) + old_mode = os.stat(dir1).st_mode + test(dir1, target_is_directory=True) + new_mode = os.stat(dir1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') + @os_helper.skip_unless_symlink + def test_cleanup_with_symlink_flags(self): + # cleanup() should not follow symlinks when fixing flags (#91133) + flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chflags(symlink, flags, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chflags(symlink, flags) + except FileNotFoundError: + pass + os.chflags(d1.name, flags) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chflags(file1, flags) + old_flags = os.stat(file1).st_flags + test(file1, target_is_directory=False) + new_flags = os.stat(file1).st_flags + self.assertEqual(new_flags, old_flags) + + with self.subTest('existing dir'): + os.chflags(dir1, flags) + old_flags = os.stat(dir1).st_flags + test(dir1, target_is_directory=True) + new_flags = os.stat(dir1).st_flags + self.assertEqual(new_flags, old_flags) + @support.cpython_only def test_del_on_collection(self): # A TemporaryDirectory is deleted when garbage collected @@ -1834,10 +1953,7 @@ def test_modes(self): d.cleanup() self.assertFalse(os.path.exists(d.name)) - @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') - def test_flags(self): - flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK - + def check_flags(self, flags): # skip the test if these flags are not supported (ex: FreeBSD 13) filename = os_helper.TESTFN try: @@ -1846,13 +1962,18 @@ def test_flags(self): os.chflags(filename, flags) except OSError as exc: # "OSError: [Errno 45] Operation not supported" - self.skipTest(f"chflags() doesn't support " - f"UF_IMMUTABLE|UF_NOUNLINK: {exc}") + self.skipTest(f"chflags() doesn't support flags " + f"{flags:#b}: {exc}") else: os.chflags(filename, 0) finally: os_helper.unlink(filename) + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') + def test_flags(self): + flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + d = self.do_create(recurse=3, dirs=2, files=2) with d: # Change files and directories flags recursively. diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 146e2dbc0fc396..3060af44fd7e3d 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -28,7 +28,7 @@ try: from test.support import interpreters -except ModuleNotFoundError: +except ImportError: interpreters = None threading_helper.requires_working_threading(module=True) @@ -1365,7 +1365,7 @@ def test_threads_join_with_no_main(self): DONE = b'D' interp = interpreters.create() - interp.run(f"""if True: + interp.exec_sync(f"""if True: import os import threading import time diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index ca99caaf88b80d..6639eaaa59936a 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -479,26 +479,46 @@ def test2(e): pass def test_unbind2(self): f = self.frame + f.wait_visibility() + f.focus_force() + f.update_idletasks() event = '' self.assertEqual(f.bind(), ()) self.assertEqual(f.bind(event), '') - def test1(e): pass - def test2(e): pass + def test1(e): events.append('a') + def test2(e): events.append('b') + def test3(e): events.append('c') funcid = f.bind(event, test1) funcid2 = f.bind(event, test2, add=True) + funcid3 = f.bind(event, test3, add=True) + events = [] + f.event_generate(event) + self.assertEqual(events, ['a', 'b', 'c']) - f.unbind(event, funcid) + f.unbind(event, funcid2) script = f.bind(event) - self.assertNotIn(funcid, script) - self.assertCommandNotExist(funcid) - self.assertCommandExist(funcid2) + self.assertNotIn(funcid2, script) + self.assertIn(funcid, script) + self.assertIn(funcid3, script) + self.assertEqual(f.bind(), (event,)) + self.assertCommandNotExist(funcid2) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid3) + events = [] + f.event_generate(event) + self.assertEqual(events, ['a', 'c']) - f.unbind(event, funcid2) + f.unbind(event, funcid) + f.unbind(event, funcid3) self.assertEqual(f.bind(event), '') self.assertEqual(f.bind(), ()) self.assertCommandNotExist(funcid) self.assertCommandNotExist(funcid2) + self.assertCommandNotExist(funcid3) + events = [] + f.event_generate(event) + self.assertEqual(events, []) # non-idempotent self.assertRaises(tkinter.TclError, f.unbind, event, funcid2) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 290f4608c5e739..21e8637a7ca905 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -615,6 +615,16 @@ def test_string(self): OP '}' (3, 0) (3, 1) FSTRING_MIDDLE '__' (3, 1) (3, 3) FSTRING_END "'" (3, 3) (3, 4) + """) + + self.check_tokenize("""\ + '''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli + aktualni pracownicy, obecni pracownicy''' +""", """\ + INDENT ' ' (1, 0) (1, 4) + STRING "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\\n aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45) + NEWLINE '\\n' (2, 45) (2, 46) + DEDENT '' (3, 0) (3, 0) """) def test_function(self): diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py index 671ec2961e7f8f..0e7ed67de71067 100644 --- a/Lib/test/test_tools/test_freeze.py +++ b/Lib/test/test_tools/test_freeze.py @@ -14,6 +14,8 @@ @support.requires_zlib() @unittest.skipIf(sys.platform.startswith('win'), 'not supported on Windows') +@unittest.skipIf(sys.platform == 'darwin' and sys._framework, + 'not supported for frameworks builds on macOS') @support.skip_if_buildbot('not all buildbots have enough space') # gh-103053: Skip test if Python is built with Profile Guided Optimization # (PGO), since the test is just too slow in this case. diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b43dca6f640b9a..a6708119b81191 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -8,6 +8,7 @@ import inspect import builtins import unittest +import unittest.mock import re import tempfile import random @@ -24,6 +25,7 @@ import json import textwrap import traceback +import contextlib from functools import partial from pathlib import Path @@ -41,6 +43,14 @@ class TracebackCases(unittest.TestCase): # For now, a very minimal set of tests. I want to be sure that # formatting of SyntaxErrors works based on changes for 2.1. + def setUp(self): + super().setUp() + self.colorize = traceback._COLORIZE + traceback._COLORIZE = False + + def tearDown(self): + super().tearDown() + traceback._COLORIZE = self.colorize def get_exception_format(self, func, exc): try: @@ -521,7 +531,7 @@ def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.print_exception)), ('(exc, /, value=, tb=, ' - 'limit=None, file=None, chain=True)')) + 'limit=None, file=None, chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception)), @@ -578,6 +588,7 @@ def f(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' if True: raise ValueError("basic caret tests")\n' ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -596,6 +607,7 @@ def f_with_unicode(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+1}, in f_with_unicode\n' ' if True: raise ValueError("Ĥellö Wörld")\n' ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -613,6 +625,7 @@ def foo(a: THIS_DOES_NOT_EXIST ) -> int: 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+1}, in f_with_type\n' ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' ' ^^^^^^^^^^^^^^^^^^^\n' @@ -633,9 +646,14 @@ def f_with_multiline(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+1}, in f_with_multiline\n' ' if True: raise ValueError(\n' - ' ^^^^^^^^^^^^^^^^^' + ' ^^^^^^^^^^^^^^^^^\n' + ' "error over multiple lines"\n' + ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + ' )\n' + ' ^' ) result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) @@ -664,9 +682,10 @@ def f_with_multiline(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' ' return compile(code, "?", "exec")\n' - ' ^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + ' ~~~~~~~^^^^^^^^^^^^^^^^^^^\n' ' File "?", line 7\n' ' foo(a, z\n' ' ^' @@ -689,9 +708,12 @@ def f_with_multiline(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' ' 2 + 1 /\n' - ' ^^^' + ' ~~^\n' + ' 0\n' + ' ~' ) result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) @@ -706,6 +728,7 @@ def f_with_binary_operator(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' ' return 10 + divisor / 0 + 30\n' ' ~~~~~~~~^~~\n' @@ -723,6 +746,7 @@ def f_with_binary_operator(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' ' return 10 + áóí / 0 + 30\n' ' ~~~~^~~\n' @@ -740,6 +764,7 @@ def f_with_binary_operator(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' ' return 10 + divisor // 0 + 30\n' ' ~~~~~~~~^^~~\n' @@ -751,16 +776,102 @@ def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): def f_with_binary_operator(): a = 1 b = "" - return ( a ) + b + return ( a ) +b lineno_f = f_with_binary_operator.__code__.co_firstlineno expected_error = ( 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' - ' return ( a ) + b\n' - ' ~~~~~~~~~~^~~\n' + ' return ( a ) +b\n' + ' ~~~~~~~~~~^~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_multiline(self): + def f_with_binary_operator(): + b = 1 + c = "" + a = b \ + +\ + c # test + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' a = b \\\n' + ' ~~~~~~\n' + ' +\\\n' + ' ^~\n' + ' c # test\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_multiline_two_char(self): + def f_with_binary_operator(): + b = 1 + c = "" + a = ( + (b # test + + ) \ + # + + << (c # test + \ + ) # test + ) + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+4}, in f_with_binary_operator\n' + ' (b # test +\n' + ' ~~~~~~~~~~~~\n' + ' ) \\\n' + ' ~~~~\n' + ' # +\n' + ' ~~~\n' + ' << (c # test\n' + ' ^^~~~~~~~~~~~\n' + ' \\\n' + ' ~\n' + ' ) # test\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_multiline_with_unicode(self): + def f_with_binary_operator(): + b = 1 + a = ("ááá" + + "áá") + b + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' a = ("ááá" +\n' + ' ~~~~~~~~\n' + ' "áá") + b\n' + ' ~~~~~~^~~\n' ) result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) @@ -775,6 +886,7 @@ def f_with_subscript(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' " return some_dict['x']['y']['z']\n" ' ~~~~~~~~~~~~~~~~~~~^^^^^\n' @@ -792,6 +904,7 @@ def f_with_subscript(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' " return some_dict['ó']['á']['í']['beta']\n" ' ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^\n' @@ -810,6 +923,7 @@ def f_with_binary_operator(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' ' return b [ a ] + c\n' ' ~~~~~~^^^^^^^^^\n' @@ -817,6 +931,226 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_subscript_multiline(self): + def f_with_subscript(): + bbbbb = {} + ccc = 1 + ddd = 2 + b = bbbbb \ + [ ccc # test + + + ddd \ + + ] # test + return b + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+4}, in f_with_subscript\n' + ' b = bbbbb \\\n' + ' ~~~~~~~\n' + ' [ ccc # test\n' + ' ^^^^^^^^^^^^^\n' + ' \n' + ' \n' + ' + ddd \\\n' + ' ^^^^^^^^\n' + ' \n' + ' \n' + ' ] # test\n' + ' ^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call(self): + def f_with_call(): + def f1(a): + def f2(b): + raise RuntimeError("fail") + return f2 + return f1("x")("y") + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' + ' return f1("x")("y")\n' + ' ~~~~~~~^^^^^\n' + f' File "{__file__}", line {lineno_f+3}, in f2\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call_unicode(self): + def f_with_call(): + def f1(a): + def f2(b): + raise RuntimeError("fail") + return f2 + return f1("ó")("á") + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' + ' return f1("ó")("á")\n' + ' ~~~~~~~^^^^^\n' + f' File "{__file__}", line {lineno_f+3}, in f2\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + def f(a): + raise RuntimeError("fail") + return f ( "x" ) + 2 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return f ( "x" ) + 2\n' + ' ~~~~~~^^^^^^^^^^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call_multiline(self): + def f_with_call(): + class C: + def y(self, a): + def f(b): + raise RuntimeError("fail") + return f + def g(x): + return C() + a = (g(1).y)( + 2 + )(3)(4) + return a + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+8}, in f_with_call\n' + ' a = (g(1).y)(\n' + ' ~~~~~~~~~\n' + ' 2\n' + ' ~\n' + ' )(3)(4)\n' + ' ~^^^\n' + f' File "{__file__}", line {lineno_f+4}, in f\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_many_lines(self): + def f(): + x = 1 + if True: x += ( + "a" + + "a" + ) # test + + lineno_f = f.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' if True: x += (\n' + ' ^^^^^^\n' + ' ...<2 lines>...\n' + ' ) # test\n' + ' ^\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_many_lines_no_caret(self): + def f(): + x = 1 + x += ( + "a" + + "a" + ) + + lineno_f = f.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' x += (\n' + ' ...<2 lines>...\n' + ' )\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_many_lines_binary_op(self): + def f_with_binary_operator(): + b = 1 + c = "a" + a = ( + b + + b + ) + ( + c + + c + + c + ) + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' a = (\n' + ' ~\n' + ' b +\n' + ' ~~~\n' + ' b\n' + ' ~\n' + ' ) + (\n' + ' ~~^~~\n' + ' c +\n' + ' ~~~\n' + ' ...<2 lines>...\n' + ' )\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + def test_traceback_specialization_with_syntax_error(self): bytecode = compile("1 / 0 / 1 / 2\n", TESTFN, "exec") @@ -833,6 +1167,7 @@ def test_traceback_specialization_with_syntax_error(self): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{TESTFN}", line {lineno_f}, in \n' " 1 $ 0 / 1 / 2\n" ' ^^^^^\n' @@ -855,6 +1190,7 @@ def test_traceback_very_long_line(self): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{TESTFN}", line {lineno_f}, in \n' f' {source}\n' f' {" "*len("if True: ") + "^"*256}\n' @@ -872,6 +1208,7 @@ def f_with_subscript(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' " some_dict['x']['y']['z']\n" ' ~~~~~~~~~~~~~~~~~~~^^^^^\n' @@ -891,6 +1228,7 @@ def exc(): f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | callable()\n' + f' | ~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 1}, in exc\n' f' | if True: raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -956,6 +1294,7 @@ def g(): pass 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_applydescs + 1}, in applydecs\n' ' @dec_error\n' ' ^^^^^^^^^\n' @@ -974,6 +1313,7 @@ class A: pass 'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' + ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_applydescs_class + 1}, in applydecs_class\n' ' @dec_error\n' ' ^^^^^^^^^\n' @@ -992,6 +1332,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", " .method", " ^^^^^^", @@ -1008,6 +1349,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", " method", ] @@ -1023,6 +1365,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", " . method", " ^^^^^^", @@ -1038,6 +1381,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", " ｗｉｄｔｈ", ] @@ -1054,6 +1398,7 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", " raise ValueError(ｗｉｄｔｈ)", ] @@ -1072,9 +1417,12 @@ def f(): "Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", + " ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 4}, in f", f" print(1, ｗｗｗ(", - f" ^^^^^^^", + f" ~~~~~~^", + f" ｔｈ))", + f" ^^^^^", ] self.assertEqual(actual, expected) @@ -1089,6 +1437,7 @@ def f(): f"Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", f" callable()", + f" ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 3}, in f", f" return 说明说明 / şçöğıĤellö", f" ~~~~~~~~~^~~~~~~~~~~~", @@ -1105,6 +1454,7 @@ def f(): f"Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", f" callable()", + f" ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", f' return "✨🐍" + func_说明说明("📗🚛",', f" ^^^^^^^^^^^^^", @@ -1127,6 +1477,7 @@ def f(): f"Traceback (most recent call last):", f" File \"{__file__}\", line {self.callable_line}, in get_exception", f" callable()", + f" ~~~~~~~~^^", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 8}, in f", f' return my_dct["✨🚛✨"]["说明"]["🐍"]["说明"]["🐍🐍"]', f" ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^", @@ -1141,6 +1492,7 @@ def f(): expected = ['Traceback (most recent call last):', f' File "{__file__}", line {self.callable_line}, in get_exception', ' callable()', + ' ~~~~~~~~^^', f' File "{__file__}", line {f.__code__.co_firstlineno + 1}, in f', ' raise MemoryError()'] self.assertEqual(actual, expected) @@ -1187,6 +1539,14 @@ class TracebackFormatMixin: def some_exception(self): raise KeyError('blah') + def _filter_debug_ranges(self, expected): + return [line for line in expected if not set(line.strip()) <= set("^~")] + + def _maybe_filter_debug_ranges(self, expected): + if not self.DEBUG_RANGES: + return self._filter_debug_ranges(expected) + return expected + @cpython_only def check_traceback_format(self, cleanup_func=None): from _testcapi import traceback_print @@ -1199,6 +1559,11 @@ def check_traceback_format(self, cleanup_func=None): cleanup_func(tb.tb_next) traceback_fmt = 'Traceback (most recent call last):\n' + \ ''.join(traceback.format_tb(tb)) + # clear caret lines from traceback_fmt since internal API does + # not emit them + traceback_fmt = "\n".join( + self._filter_debug_ranges(traceback_fmt.splitlines()) + ) + "\n" file_ = StringIO() traceback_print(tb, file_) python_fmt = file_.getvalue() @@ -1291,12 +1656,16 @@ def f(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_f+5}, in _check_recursive_traceback_display\n' ' f()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' f()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' f()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' f()\n' + ' ~^^\n' # XXX: The following line changes depending on whether the tests # are run through the interactive interpreter or with -m # It also varies depending on the platform (stack size) @@ -1305,7 +1674,7 @@ def f(): 'RecursionError: maximum recursion depth exceeded\n' ) - expected = result_f.splitlines() + expected = self._maybe_filter_debug_ranges(result_f.splitlines()) actual = stderr_f.getvalue().splitlines() # Check the output text matches expectations @@ -1337,13 +1706,13 @@ def g(count=10): result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -1353,11 +1722,10 @@ def g(count=10): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_g+7}, in _check_recursive_traceback_display\n' ' g()\n' + ' ~^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() - if not self.DEBUG_RANGES: - expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) # Check 2 different repetitive sections @@ -1379,23 +1747,23 @@ def h(count=10): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_h+7}, in _check_recursive_traceback_display\n' ' h()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_h+3}, in h\n' ' g()\n' + ' ~^^\n' ) - expected = (result_h + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((result_h + result_g).splitlines()) actual = stderr_h.getvalue().splitlines() - if not self.DEBUG_RANGES: - expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) # Check the boundary conditions. First, test just below the cutoff. @@ -1409,26 +1777,25 @@ def h(count=10): result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' 'ValueError\n' ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+81}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+80}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF)\n' + ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() - if not self.DEBUG_RANGES: - expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) # Second, test just above the cutoff. @@ -1442,13 +1809,13 @@ def h(count=10): result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' ' return g(count-1)\n' - ' ^^^^^^^^^^\n' + ' ~^^^^^^^^^\n' ' [Previous line repeated 1 more time]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -1456,13 +1823,12 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+114}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+112}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF + 1)\n' + ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() - if not self.DEBUG_RANGES: - expected = [line for line in expected if not set(line.strip()) == {"^"}] self.assertEqual(actual, expected) @requires_debug_ranges() @@ -1942,6 +2308,7 @@ def exc(): f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 1}, in exc\n' f' | raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])\n' f' | ExceptionGroup: eg (2 sub-exceptions)\n' @@ -1977,6 +2344,7 @@ def exc(): f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' f' | raise EG("eg2", [ValueError(3), TypeError(4)]) from e\n' f' | ExceptionGroup: eg2 (2 sub-exceptions)\n' @@ -2028,6 +2396,7 @@ def exc(): f'Traceback (most recent call last):\n' f' File "{__file__}", line {self.callable_line}, in get_exception\n' f' exception_or_callable()\n' + f' ~~~~~~~~~~~~~~~~~~~~~^^\n' f' File "{__file__}", line {exc.__code__.co_firstlineno + 8}, in exc\n' f' raise ImportError(5)\n' f'ImportError: 5\n') @@ -2074,6 +2443,7 @@ def exc(): f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 11}, in exc\n' f' | raise EG("top", [VE(5)])\n' f' | ExceptionGroup: top (1 sub-exception)\n' @@ -2233,6 +2603,7 @@ def exc(): expected = (f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 9}, in exc\n' f' | raise ExceptionGroup("nested", excs)\n' f' | ExceptionGroup: nested (2 sub-exceptions)\n' @@ -2284,6 +2655,7 @@ def exc(): expected = (f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 10}, in exc\n' f' | raise ExceptionGroup("nested", excs)\n' f' | ExceptionGroup: nested (2 sub-exceptions)\n' @@ -2552,7 +2924,7 @@ def test_basics(self): def test_lazy_lines(self): linecache.clearcache() f = traceback.FrameSummary("f", 1, "dummy", lookup_line=False) - self.assertEqual(None, f._line) + self.assertEqual(None, f._lines) linecache.lazycache("f", globals()) self.assertEqual( '"""Test cases for traceback module"""', @@ -2669,7 +3041,7 @@ def some_inner(k, v): def test_custom_format_frame(self): class CustomStackSummary(traceback.StackSummary): - def format_frame_summary(self, frame_summary): + def format_frame_summary(self, frame_summary, colorize=False): return f'{frame_summary.filename}:{frame_summary.lineno}' def some_inner(): @@ -2694,7 +3066,7 @@ def g(): tb = g() class Skip_G(traceback.StackSummary): - def format_frame_summary(self, frame_summary): + def format_frame_summary(self, frame_summary, colorize=False): if frame_summary.name == 'g': return None return super().format_frame_summary(frame_summary) @@ -2714,10 +3086,9 @@ def __repr__(self) -> str: raise Exception("Unrepresentable") class TestTracebackException(unittest.TestCase): - - def test_smoke(self): + def do_test_smoke(self, exc, expected_type_str): try: - 1/0 + raise exc except Exception as e: exc_obj = e exc = traceback.TracebackException.from_exception(e) @@ -2727,9 +3098,23 @@ def test_smoke(self): self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(expected_type_str, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) + def test_smoke_builtin(self): + self.do_test_smoke(ValueError(42), 'ValueError') + + def test_smoke_user_exception(self): + class MyException(Exception): + pass + + self.do_test_smoke( + MyException('bad things happened'), + ('test.test_traceback.TestTracebackException.' + 'test_smoke_user_exception..MyException')) + def test_from_exception(self): # Check all the parameters are accepted. def foo(): @@ -2750,7 +3135,9 @@ def foo(): self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_cause(self): @@ -2772,7 +3159,9 @@ def test_cause(self): self.assertEqual(exc_context, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_context(self): @@ -2792,7 +3181,9 @@ def test_context(self): self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_long_context_chain(self): @@ -2837,7 +3228,9 @@ def test_compact_with_cause(self): self.assertEqual(None, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) def test_compact_no_cause(self): @@ -2857,9 +3250,22 @@ def test_compact_no_cause(self): self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(type(exc_obj), exc.exc_type) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) + def test_no_save_exc_type(self): + try: + 1/0 + except Exception as e: + exc = e + + te = traceback.TracebackException.from_exception( + exc, save_exc_type=False) + with self.assertWarns(DeprecationWarning): + self.assertIsNone(te.exc_type) + def test_no_refs_to_exception_and_traceback_objects(self): try: 1/0 @@ -3108,6 +3514,7 @@ def test_exception_group_format(self): f' | Traceback (most recent call last):', f' | File "{__file__}", line {lno_g+9}, in _get_exception_group', f' | f()', + f' | ~^^', f' | File "{__file__}", line {lno_f+1}, in f', f' | 1/0', f' | ~^~', @@ -3116,6 +3523,7 @@ def test_exception_group_format(self): f' | Traceback (most recent call last):', f' | File "{__file__}", line {lno_g+13}, in _get_exception_group', f' | g(42)', + f' | ~^^^^', f' | File "{__file__}", line {lno_g+1}, in g', f' | raise ValueError(v)', f' | ValueError: 42', @@ -3124,6 +3532,7 @@ def test_exception_group_format(self): f' | Traceback (most recent call last):', f' | File "{__file__}", line {lno_g+20}, in _get_exception_group', f' | g(24)', + f' | ~^^^^', f' | File "{__file__}", line {lno_g+1}, in g', f' | raise ValueError(v)', f' | ValueError: 24', @@ -3845,6 +4254,115 @@ def test_levenshtein_distance_short_circuit(self): res3 = traceback._levenshtein_distance(a, b, threshold) self.assertGreater(res3, threshold, msg=(a, b, threshold)) +class TestColorizedTraceback(unittest.TestCase): + def test_colorized_traceback(self): + def foo(*args): + x = {'a':{'b': None}} + y = x['a']['b']['c'] + + def baz(*args): + return foo(1,2,3,4) + + def bar(): + return baz(1, + 2,3 + ,4) + try: + bar() + except Exception as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + lines = "".join(exc.format(colorize=True)) + red = traceback._ANSIColors.RED + boldr = traceback._ANSIColors.BOLD_RED + reset = traceback._ANSIColors.RESET + self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) + self.assertIn("return " + red + "foo" + reset + boldr + "(1,2,3,4)" + reset, lines) + self.assertIn("return " + red + "baz" + reset + boldr + "(1," + reset, lines) + self.assertIn(boldr + "2,3" + reset, lines) + self.assertIn(boldr + ",4)" + reset, lines) + self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) + + def test_colorized_syntax_error(self): + try: + compile("a $ b", "", "exec") + except SyntaxError as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + actual = "".join(exc.format(colorize=True)) + red = traceback._ANSIColors.RED + magenta = traceback._ANSIColors.MAGENTA + boldm = traceback._ANSIColors.BOLD_MAGENTA + boldr = traceback._ANSIColors.BOLD_RED + reset = traceback._ANSIColors.RESET + expected = "".join([ + f' File {magenta}""{reset}, line {magenta}1{reset}\n', + f' a {boldr}${reset} b\n', + f' {boldr}^{reset}\n', + f'{boldm}SyntaxError{reset}: {magenta}invalid syntax{reset}\n'] + ) + self.assertIn(expected, actual) + + def test_colorized_traceback_is_the_default(self): + def foo(): + 1/0 + + from _testcapi import exception_print + try: + foo() + self.fail("No exception thrown.") + except Exception as e: + with captured_output("stderr") as tbstderr: + with unittest.mock.patch('traceback._can_colorize', return_value=True): + exception_print(e) + actual = tbstderr.getvalue().splitlines() + + red = traceback._ANSIColors.RED + boldr = traceback._ANSIColors.BOLD_RED + magenta = traceback._ANSIColors.MAGENTA + boldm = traceback._ANSIColors.BOLD_MAGENTA + reset = traceback._ANSIColors.RESET + lno_foo = foo.__code__.co_firstlineno + expected = ['Traceback (most recent call last):', + f' File {magenta}"{__file__}"{reset}, ' + f'line {magenta}{lno_foo+5}{reset}, in {magenta}test_colorized_traceback_is_the_default{reset}', + f' {red}foo{reset+boldr}(){reset}', + f' {red}~~~{reset+boldr}^^{reset}', + f' File {magenta}"{__file__}"{reset}, ' + f'line {magenta}{lno_foo+1}{reset}, in {magenta}foo{reset}', + f' {red}1{reset+boldr}/{reset+red}0{reset}', + f' {red}~{reset+boldr}^{reset+red}~{reset}', + f'{boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}'] + self.assertEqual(actual, expected) + + def test_colorized_detection_checks_for_environment_variables(self): + if sys.platform == "win32": + virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", return_value=True) + else: + virtual_patching = contextlib.nullcontext() + with virtual_patching: + with unittest.mock.patch("os.isatty") as isatty_mock: + isatty_mock.return_value = True + with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): + self.assertEqual(traceback._can_colorize(), False) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): + self.assertEqual(traceback._can_colorize(), True) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): + self.assertEqual(traceback._can_colorize(), False) + with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): + self.assertEqual(traceback._can_colorize(), False) + with unittest.mock.patch("os.environ", {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): + self.assertEqual(traceback._can_colorize(), True) + with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): + self.assertEqual(traceback._can_colorize(), True) + with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): + self.assertEqual(traceback._can_colorize(), False) + with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): + self.assertEqual(traceback._can_colorize(), False) + isatty_mock.return_value = False + self.assertEqual(traceback._can_colorize(), False) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 52c4b7a0beabd0..9a04a95dc40d65 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -258,6 +258,55 @@ def test_element_create_image_errors(self): with self.assertRaisesRegex(TclError, 'bad option'): style.element_create('block2', 'image', image, spam=1) + def test_element_create_vsapi_1(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + style.element_create('smallclose', 'vsapi', 'WINDOW', 19, [ + ('disabled', 4), + ('pressed', 3), + ('active', 2), + ('', 1)]) + style.layout('CloseButton', + [('CloseButton.smallclose', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='CloseButton') + b.pack(expand=True, fill='both') + self.assertEqual(b.winfo_reqwidth(), 13) + self.assertEqual(b.winfo_reqheight(), 13) + + def test_element_create_vsapi_2(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + style.element_create('pin', 'vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]) + style.layout('Explorer.Pin', + [('Explorer.Pin.pin', {'sticky': 'news'})]) + pin = ttk.Checkbutton(self.root, style='Explorer.Pin') + pin.pack(expand=True, fill='both') + self.assertEqual(pin.winfo_reqwidth(), 16) + self.assertEqual(pin.winfo_reqheight(), 16) + + def test_element_create_vsapi_3(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + style.element_create('headerclose', 'vsapi', 'EXPLORERBAR', 2, [ + ('pressed', 3), + ('active', 2), + ('', 1)]) + style.layout('Explorer.CloseButton', + [('Explorer.CloseButton.headerclose', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='Explorer.CloseButton') + b.pack(expand=True, fill='both') + self.assertEqual(b.winfo_reqwidth(), 16) + self.assertEqual(b.winfo_reqheight(), 16) + def test_theme_create(self): style = self.style curr_theme = style.theme_use() @@ -358,6 +407,39 @@ def test_theme_create_image(self): style.theme_use(curr_theme) + def test_theme_create_vsapi(self): + style = self.style + if 'xpnative' not in style.theme_names(): + self.skipTest("requires 'xpnative' theme") + curr_theme = style.theme_use() + new_theme = 'testtheme5' + style.theme_create(new_theme, settings={ + 'pin' : { + 'element create': ['vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]], + }, + 'Explorer.Pin' : { + 'layout': [('Explorer.Pin.pin', {'sticky': 'news'})], + }, + }) + + style.theme_use(new_theme) + self.assertIn('pin', style.element_names()) + self.assertEqual(style.layout('Explorer.Pin'), + [('Explorer.Pin.pin', {'sticky': 'nswe'})]) + + pin = ttk.Checkbutton(self.root, style='Explorer.Pin') + pin.pack(expand=True, fill='both') + self.assertEqual(pin.winfo_reqwidth(), 16) + self.assertEqual(pin.winfo_reqheight(), 16) + + style.theme_use(curr_theme) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ttk_textonly.py b/Lib/test/test_ttk_textonly.py index 96dc179a69ecac..e6525c4d6c6982 100644 --- a/Lib/test/test_ttk_textonly.py +++ b/Lib/test/test_ttk_textonly.py @@ -179,7 +179,7 @@ def test_format_elemcreate(self): # don't format returned values as a tcl script # minimum acceptable for image type self.assertEqual(ttk._format_elemcreate('image', False, 'test'), - ("test ", ())) + ("test", ())) # specifying a state spec self.assertEqual(ttk._format_elemcreate('image', False, 'test', ('', 'a')), ("test {} a", ())) @@ -203,17 +203,19 @@ def test_format_elemcreate(self): # don't format returned values as a tcl script # minimum acceptable for vsapi self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b'), - ("a b ", ())) + ('a', 'b', ('', 1), ())) # now with a state spec with multiple states self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', - ('a', 'b', 'c')), ("a b {a b} c", ())) + [('a', 'b', 'c')]), ('a', 'b', ('a b', 'c'), ())) # state spec and option self.assertEqual(ttk._format_elemcreate('vsapi', False, 'a', 'b', - ('a', 'b'), opt='x'), ("a b a b", ("-opt", "x"))) + [('a', 'b')], opt='x'), ('a', 'b', ('a', 'b'), ("-opt", "x"))) # format returned values as a tcl script # state spec with a multivalue and an option self.assertEqual(ttk._format_elemcreate('vsapi', True, 'a', 'b', - ('a', 'b', [1, 2]), opt='x'), ("{a b {a b} {1 2}}", "-opt x")) + opt='x'), ("a b {{} 1}", "-opt x")) + self.assertEqual(ttk._format_elemcreate('vsapi', True, 'a', 'b', + [('a', 'b', [1, 2])], opt='x'), ("a b {{a b} {1 2}}", "-opt x")) # Testing type = from # from type expects at least a type name @@ -222,9 +224,9 @@ def test_format_elemcreate(self): self.assertEqual(ttk._format_elemcreate('from', False, 'a'), ('a', ())) self.assertEqual(ttk._format_elemcreate('from', False, 'a', 'b'), - ('a', ('b', ))) + ('a', ('b',))) self.assertEqual(ttk._format_elemcreate('from', True, 'a', 'b'), - ('{a}', 'b')) + ('a', 'b')) def test_format_layoutlist(self): @@ -326,6 +328,22 @@ def test_script_from_settings(self): "ttk::style element create thing image {name {state1 state2} val} " "-opt {3 2m}") + vsapi = {'pin': {'element create': + ['vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]]}} + self.assertEqual(ttk._script_from_settings(vsapi), + "ttk::style element create pin vsapi EXPLORERBAR 3 {" + "{pressed !selected} 3 " + "{active !selected} 2 " + "{pressed selected} 6 " + "{active selected} 5 " + "selected 4 " + "{} 1} ") def test_tclobj_to_py(self): self.assertEqual( diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index da32c4ea6477ce..bfecd8eb71220c 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1,6 +1,6 @@ # Python test set -- part 6, built-in types -from test.support import run_with_locale, cpython_only +from test.support import run_with_locale, cpython_only, MISSING_C_DOCSTRINGS import collections.abc from collections import namedtuple import copy @@ -598,6 +598,8 @@ def test_slot_wrapper_types(self): self.assertIsInstance(object.__lt__, types.WrapperDescriptorType) self.assertIsInstance(int.__lt__, types.WrapperDescriptorType) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_dunder_get_signature(self): sig = inspect.signature(object.__init__.__get__) self.assertEqual(list(sig.parameters), ["instance", "owner"]) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 31d7fda2f978da..2d10c39840ddf3 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3533,13 +3533,26 @@ def __subclasshook__(cls, other): def test_issubclass_fails_correctly(self): @runtime_checkable - class P(Protocol): + class NonCallableMembers(Protocol): x = 1 + class NotRuntimeCheckable(Protocol): + def callable_member(self) -> int: ... + + @runtime_checkable + class RuntimeCheckable(Protocol): + def callable_member(self) -> int: ... + class C: pass - with self.assertRaisesRegex(TypeError, r"issubclass\(\) arg 1 must be a class"): - issubclass(C(), P) + # These three all exercise different code paths, + # but should result in the same error message: + for protocol in NonCallableMembers, NotRuntimeCheckable, RuntimeCheckable: + with self.subTest(proto_name=protocol.__name__): + with self.assertRaisesRegex( + TypeError, r"issubclass\(\) arg 1 must be a class" + ): + issubclass(C(), protocol) def test_defining_generic_protocols(self): T = TypeVar('T') @@ -7535,6 +7548,83 @@ class GenericNamedTuple(NamedTuple, Generic[T]): self.assertEqual(CallNamedTuple.__orig_bases__, (NamedTuple,)) + def test_setname_called_on_values_in_class_dictionary(self): + class Vanilla: + def __set_name__(self, owner, name): + self.name = name + + class Foo(NamedTuple): + attr = Vanilla() + + foo = Foo() + self.assertEqual(len(foo), 0) + self.assertNotIn('attr', Foo._fields) + self.assertIsInstance(foo.attr, Vanilla) + self.assertEqual(foo.attr.name, "attr") + + class Bar(NamedTuple): + attr: Vanilla = Vanilla() + + bar = Bar() + self.assertEqual(len(bar), 1) + self.assertIn('attr', Bar._fields) + self.assertIsInstance(bar.attr, Vanilla) + self.assertEqual(bar.attr.name, "attr") + + def test_setname_raises_the_same_as_on_other_classes(self): + class CustomException(BaseException): pass + + class Annoying: + def __set_name__(self, owner, name): + raise CustomException + + annoying = Annoying() + + with self.assertRaises(CustomException) as cm: + class NormalClass: + attr = annoying + normal_exception = cm.exception + + with self.assertRaises(CustomException) as cm: + class NamedTupleClass(NamedTuple): + attr = annoying + namedtuple_exception = cm.exception + + self.assertIs(type(namedtuple_exception), CustomException) + self.assertIs(type(namedtuple_exception), type(normal_exception)) + + self.assertEqual(len(namedtuple_exception.__notes__), 1) + self.assertEqual( + len(namedtuple_exception.__notes__), len(normal_exception.__notes__) + ) + + expected_note = ( + "Error calling __set_name__ on 'Annoying' instance " + "'attr' in 'NamedTupleClass'" + ) + self.assertEqual(namedtuple_exception.__notes__[0], expected_note) + self.assertEqual( + namedtuple_exception.__notes__[0], + normal_exception.__notes__[0].replace("NormalClass", "NamedTupleClass") + ) + + def test_strange_errors_when_accessing_set_name_itself(self): + class CustomException(Exception): pass + + class Meta(type): + def __getattribute__(self, attr): + if attr == "__set_name__": + raise CustomException + return object.__getattribute__(self, attr) + + class VeryAnnoying(metaclass=Meta): pass + + very_annoying = VeryAnnoying() + + with self.assertRaises(CustomException): + class Foo(NamedTuple): + attr = very_annoying + class TypedDictTests(BaseTestCase): def test_basics_functional_syntax(self): @@ -7692,6 +7782,46 @@ class Cat(Animal): 'voice': str, }) + def test_keys_inheritance_with_same_name(self): + class NotTotal(TypedDict, total=False): + a: int + + class Total(NotTotal): + a: int + + self.assertEqual(NotTotal.__required_keys__, frozenset()) + self.assertEqual(NotTotal.__optional_keys__, frozenset(['a'])) + self.assertEqual(Total.__required_keys__, frozenset(['a'])) + self.assertEqual(Total.__optional_keys__, frozenset()) + + class Base(TypedDict): + a: NotRequired[int] + b: Required[int] + + class Child(Base): + a: Required[int] + b: NotRequired[int] + + self.assertEqual(Base.__required_keys__, frozenset(['b'])) + self.assertEqual(Base.__optional_keys__, frozenset(['a'])) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset(['b'])) + + def test_multiple_inheritance_with_same_key(self): + class Base1(TypedDict): + a: NotRequired[int] + + class Base2(TypedDict): + a: Required[str] + + class Child(Base1, Base2): + pass + + # Last base wins + self.assertEqual(Child.__annotations__, {'a': Required[str]}) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset()) + def test_required_notrequired_keys(self): self.assertEqual(NontotalMovie.__required_keys__, frozenset({"title"})) @@ -8558,6 +8688,40 @@ class X(Annotated[int, (1, 10)]): ... self.assertEqual(X.__mro__, (X, int, object), "Annotated should be transparent.") + def test_annotated_cached_with_types(self): + class A(str): ... + class B(str): ... + + field_a1 = Annotated[str, A("X")] + field_a2 = Annotated[str, B("X")] + a1_metadata = field_a1.__metadata__[0] + a2_metadata = field_a2.__metadata__[0] + + self.assertIs(type(a1_metadata), A) + self.assertEqual(a1_metadata, A("X")) + self.assertIs(type(a2_metadata), B) + self.assertEqual(a2_metadata, B("X")) + self.assertIsNot(type(a1_metadata), type(a2_metadata)) + + field_b1 = Annotated[str, A("Y")] + field_b2 = Annotated[str, B("Y")] + b1_metadata = field_b1.__metadata__[0] + b2_metadata = field_b2.__metadata__[0] + + self.assertIs(type(b1_metadata), A) + self.assertEqual(b1_metadata, A("Y")) + self.assertIs(type(b2_metadata), B) + self.assertEqual(b2_metadata, B("Y")) + self.assertIsNot(type(b1_metadata), type(b2_metadata)) + + field_c1 = Annotated[int, 1] + field_c2 = Annotated[int, 1.0] + field_c3 = Annotated[int, True] + + self.assertIs(type(field_c1.__metadata__[0]), int) + self.assertIs(type(field_c2.__metadata__[0]), float) + self.assertIs(type(field_c3.__metadata__[0]), bool) + class TypeAliasTests(BaseTestCase): def test_canonical_usage_with_variable_annotation(self): diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 890672c5d27eec..8ecb23ff384362 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -47,13 +47,18 @@ def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding=encoding) + env={**os.environ, "PYTHONHOME": ""}) out, err = p.communicate() if p.returncode: if verbose and err: - print(err.decode('utf-8', 'backslashreplace')) + print(err.decode(encoding or 'utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) + if encoding: + return ( + out.decode(encoding, 'backslashreplace'), + err.decode(encoding, 'backslashreplace'), + ) return out, err class BaseTest(unittest.TestCase): @@ -281,8 +286,18 @@ def test_sysconfig(self): ('get_config_h_filename()', sysconfig.get_config_h_filename())): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call - out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), err) + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) @requireVenvCreate @unittest.skipUnless(can_symlink(), 'Needs symlinks') @@ -303,8 +318,18 @@ def test_sysconfig_symlinks(self): ('get_config_h_filename()', sysconfig.get_config_h_filename())): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call - out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), err) + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) if sys.platform == 'win32': ENV_SUBDIRS = ( diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 2c523230e7e97f..50b0f3fff04c57 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -5,6 +5,8 @@ import re import sys import textwrap +import types +from typing import overload, get_overloads import unittest from test import support from test.support import import_helper @@ -16,6 +18,7 @@ from test.test_warnings.data import stacklevel as warning_tests import warnings as original_warnings +from warnings import deprecated py_warnings = import_helper.import_fresh_module('warnings', @@ -90,7 +93,7 @@ def test_module_all_attribute(self): self.assertTrue(hasattr(self.module, '__all__')) target_api = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", - "resetwarnings", "catch_warnings"] + "resetwarnings", "catch_warnings", "deprecated"] self.assertSetEqual(set(self.module.__all__), set(target_api)) @@ -372,6 +375,28 @@ def test_append_duplicate(self): "appended duplicate changed order of filters" ) + def test_argument_validation(self): + with self.assertRaises(ValueError): + self.module.filterwarnings(action='foo') + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', message=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', category=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', category=int) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', module=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', lineno=int) + with self.assertRaises(ValueError): + self.module.filterwarnings('ignore', lineno=-1) + with self.assertRaises(ValueError): + self.module.simplefilter(action='foo') + with self.assertRaises(TypeError): + self.module.simplefilter('ignore', lineno=int) + with self.assertRaises(ValueError): + self.module.simplefilter('ignore', lineno=-1) + def test_catchwarnings_with_simplefilter_ignore(self): with original_warnings.catch_warnings(module=self.module): self.module.resetwarnings() @@ -1235,8 +1260,8 @@ def test_conflicting_envvar_and_command_line(self): b" File \"\", line 1, in ", b' import sys, warnings; sys.stdout.write(str(sys.warnoptions)); warnings.w' b"arn('Message', DeprecationWarning)", - b' ^^^^^^^^^^' - b'^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', + b' ~~~~~~~~~~' + b'~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', b"DeprecationWarning: Message"]) def test_default_filter_configuration(self): @@ -1377,6 +1402,283 @@ def test_late_resource_warning(self): self.assertTrue(err.startswith(expected), ascii(err)) +class DeprecatedTests(unittest.TestCase): + def test_dunder_deprecated(self): + @deprecated("A will go away soon") + class A: + pass + + self.assertEqual(A.__deprecated__, "A will go away soon") + self.assertIsInstance(A, type) + + @deprecated("b will go away soon") + def b(): + pass + + self.assertEqual(b.__deprecated__, "b will go away soon") + self.assertIsInstance(b, types.FunctionType) + + @overload + @deprecated("no more ints") + def h(x: int) -> int: ... + @overload + def h(x: str) -> str: ... + def h(x): + return x + + overloads = get_overloads(h) + self.assertEqual(len(overloads), 2) + self.assertEqual(overloads[0].__deprecated__, "no more ints") + + def test_class(self): + @deprecated("A will go away soon") + class A: + pass + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + A() + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + with self.assertRaises(TypeError): + A(42) + + def test_class_with_init(self): + @deprecated("HasInit will go away soon") + class HasInit: + def __init__(self, x): + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasInit will go away soon"): + instance = HasInit(42) + self.assertEqual(instance.x, 42) + + def test_class_with_new(self): + has_new_called = False + + @deprecated("HasNew will go away soon") + class HasNew: + def __new__(cls, x): + nonlocal has_new_called + has_new_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasNew will go away soon"): + instance = HasNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(has_new_called) + + def test_class_with_inherited_new(self): + new_base_called = False + + class NewBase: + def __new__(cls, x): + nonlocal new_base_called + new_base_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + @deprecated("HasInheritedNew will go away soon") + class HasInheritedNew(NewBase): + pass + + with self.assertWarnsRegex(DeprecationWarning, "HasInheritedNew will go away soon"): + instance = HasInheritedNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_base_called) + + def test_class_with_new_but_no_init(self): + new_called = False + + @deprecated("HasNewNoInit will go away soon") + class HasNewNoInit: + def __new__(cls, x): + nonlocal new_called + new_called = True + obj = super().__new__(cls) + obj.x = x + return obj + + with self.assertWarnsRegex(DeprecationWarning, "HasNewNoInit will go away soon"): + instance = HasNewNoInit(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_called) + + def test_mixin_class(self): + @deprecated("Mixin will go away soon") + class Mixin: + pass + + class Base: + def __init__(self, a) -> None: + self.a = a + + with self.assertWarnsRegex(DeprecationWarning, "Mixin will go away soon"): + class Child(Base, Mixin): + pass + + instance = Child(42) + self.assertEqual(instance.a, 42) + + def test_existing_init_subclass(self): + @deprecated("C will go away soon") + class C: + def __init_subclass__(cls) -> None: + cls.inited = True + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C): + pass + + self.assertTrue(D.inited) + self.assertIsInstance(D(), D) # no deprecation + + def test_existing_init_subclass_in_base(self): + class Base: + def __init_subclass__(cls, x) -> None: + cls.inited = x + + @deprecated("C will go away soon") + class C(Base, x=42): + pass + + self.assertEqual(C.inited, 42) + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C, x=3): + pass + + self.assertEqual(D.inited, 3) + + def test_init_subclass_has_correct_cls(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_init_subclass_with_explicit_classmethod(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + @classmethod + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_function(self): + @deprecated("b will go away soon") + def b(): + pass + + with self.assertWarnsRegex(DeprecationWarning, "b will go away soon"): + b() + + def test_method(self): + class Capybara: + @deprecated("x will go away soon") + def x(self): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x() + + def test_property(self): + class Capybara: + @property + @deprecated("x will go away soon") + def x(self): + pass + + @property + def no_more_setting(self): + return 42 + + @no_more_setting.setter + @deprecated("no more setting") + def no_more_setting(self, value): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + self.assertEqual(instance.no_more_setting, 42) + + with self.assertWarnsRegex(DeprecationWarning, "no more setting"): + instance.no_more_setting = 42 + + def test_category(self): + @deprecated("c will go away soon", category=RuntimeWarning) + def c(): + pass + + with self.assertWarnsRegex(RuntimeWarning, "c will go away soon"): + c() + + def test_turn_off_warnings(self): + @deprecated("d will go away soon", category=None) + def d(): + pass + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + d() + + def test_only_strings_allowed(self): + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'type'" + ): + @deprecated + class Foo: ... + + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'function'" + ): + @deprecated + def foo(): ... + + def test_no_retained_references_to_wrapper_instance(self): + @deprecated('depr') + def d(): pass + + self.assertFalse(any( + isinstance(cell.cell_contents, deprecated) for cell in d.__closure__ + )) + def setUpModule(): py_warnings.onceregistry.clear() c_warnings.onceregistry.clear() diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 2d695bc883131f..ca481c57c3d972 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -272,6 +272,17 @@ def test_register_preferred(self): self._check_registration(preferred=True) + @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") + def test_no_xdg_settings_on_macOS(self): + # On macOS webbrowser should not use xdg-settings to + # look for X11 based browsers (for those users with + # XQuartz installed) + with mock.patch("subprocess.check_output") as ck_o: + webbrowser.register_standard_browsers() + + ck_o.assert_not_called() + + class ImportTest(unittest.TestCase): def test_register(self): webbrowser = import_helper.import_fresh_module('webbrowser') diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index 3445702846d8a0..bf8c52e646dc18 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -1,17 +1,29 @@ # Test the internal _wmi module on Windows # This is used by the platform module, and potentially others +import time import unittest -from test.support import import_helper, requires_resource +from test.support import import_helper, requires_resource, LOOPBACK_TIMEOUT # Do this first so test will be skipped if module doesn't exist _wmi = import_helper.import_module('_wmi', required_on=['win']) +def wmi_exec_query(query): + # gh-112278: WMI maybe slow response when first call. + try: + return _wmi.exec_query(query) + except WindowsError as e: + if e.winerror != 258: + raise + time.sleep(LOOPBACK_TIMEOUT) + return _wmi.exec_query(query) + + class WmiTests(unittest.TestCase): def test_wmi_query_os_version(self): - r = _wmi.exec_query("SELECT Version FROM Win32_OperatingSystem").split("\0") + r = wmi_exec_query("SELECT Version FROM Win32_OperatingSystem").split("\0") self.assertEqual(1, len(r)) k, eq, v = r[0].partition("=") self.assertEqual("=", eq, r[0]) @@ -28,7 +40,7 @@ def test_wmi_query_repeated(self): def test_wmi_query_error(self): # Invalid queries fail with OSError try: - _wmi.exec_query("SELECT InvalidColumnName FROM InvalidTableName") + wmi_exec_query("SELECT InvalidColumnName FROM InvalidTableName") except OSError as ex: if ex.winerror & 0xFFFFFFFF == 0x80041010: # This is the expected error code. All others should fail the test @@ -42,7 +54,7 @@ def test_wmi_query_repeated_error(self): def test_wmi_query_not_select(self): # Queries other than SELECT are blocked to avoid potential exploits with self.assertRaises(ValueError): - _wmi.exec_query("not select, just in case someone tries something") + wmi_exec_query("not select, just in case someone tries something") @requires_resource('cpu') def test_wmi_query_overflow(self): @@ -50,11 +62,11 @@ def test_wmi_query_overflow(self): # Test multiple times to ensure consistency for _ in range(2): with self.assertRaises(OSError): - _wmi.exec_query("SELECT * FROM CIM_DataFile") + wmi_exec_query("SELECT * FROM CIM_DataFile") def test_wmi_query_multiple_rows(self): # Multiple instances should have an extra null separator - r = _wmi.exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") + r = wmi_exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") self.assertFalse(r.startswith("\0"), r) self.assertFalse(r.endswith("\0"), r) it = iter(r.split("\0")) @@ -69,6 +81,6 @@ def test_wmi_query_threads(self): from concurrent.futures import ThreadPoolExecutor query = "SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000" with ThreadPoolExecutor(4) as pool: - task = [pool.submit(_wmi.exec_query, query) for _ in range(32)] + task = [pool.submit(wmi_exec_query, query) for _ in range(32)] for t in task: self.assertRegex(t.result(), "ProcessId=") diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b9e7937b0bbc00..80ee064896f59a 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2535,7 +2535,7 @@ def __eq__(self, o): e.extend([ET.Element('bar')]) self.assertRaises(ValueError, e.remove, X('baz')) - @support.infinite_recursion(25) + @support.infinite_recursion() def test_recursive_repr(self): # Issue #25455 e = ET.Element('foo') diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 1dc8b91a453f92..ef02c64f886f8a 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -18,6 +18,21 @@ hasattr(zlib.decompressobj(), "copy"), 'requires Decompress.copy()') + +def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): + # Register "1.2.3" as "1.2.3.0" + # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" + v = zlib_version.split('-', 1)[0].split('.') + if len(v) < 4: + v.append('0') + elif not v[-1].isnumeric(): + v[-1] = '0' + return tuple(map(int, v)) + + +ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() + + # bpo-46623: On s390x, when a hardware accelerator is used, using different # ways to compress data with zlib can produce different compressed data. # Simplified test_pair() code: @@ -473,9 +488,8 @@ def test_flushes(self): sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', 'Z_PARTIAL_FLUSH'] - ver = tuple(int(v) for v in zlib.ZLIB_RUNTIME_VERSION.split('.')) # Z_BLOCK has a known failure prior to 1.2.5.3 - if ver >= (1, 2, 5, 3): + if ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 5, 3): sync_opt.append('Z_BLOCK') sync_opt = [getattr(zlib, opt) for opt in sync_opt @@ -793,16 +807,7 @@ def test_large_unconsumed_tail(self, size): def test_wbits(self): # wbits=0 only supported since zlib v1.2.3.5 - # Register "1.2.3" as "1.2.3.0" - # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" - v = zlib.ZLIB_RUNTIME_VERSION.split('-', 1)[0].split('.') - if len(v) < 4: - v.append('0') - elif not v[-1].isnumeric(): - v[-1] = '0' - - v = tuple(map(int, v)) - supports_wbits_0 = v >= (1, 2, 3, 5) + supports_wbits_0 = ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 3, 5) co = zlib.compressobj(level=1, wbits=15) zlib15 = co.compress(HAMLET_SCENE) + co.flush() diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 3766ceac8385f2..7b6b69d0109d88 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -17,6 +17,7 @@ from datetime import date, datetime, time, timedelta, timezone from functools import cached_property +from test.support import MISSING_C_DOCSTRINGS from test.test_zoneinfo import _support as test_support from test.test_zoneinfo._support import OS_ENV_LOCK, TZPATH_TEST_LOCK, ZoneInfoTestBase from test.support.import_helper import import_module @@ -404,6 +405,8 @@ def test_time_fixed_offset(self): class CZoneInfoTest(ZoneInfoTest): module = c_zoneinfo + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_signatures(self): """Ensure that C module has valid method signatures.""" import inspect diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 0df7f9d889413c..124882420c255c 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1527,10 +1527,24 @@ def bind(self, sequence=None, func=None, add=None): return self._bind(('bind', self._w), sequence, func, add) def unbind(self, sequence, funcid=None): - """Unbind for this widget for event SEQUENCE the - function identified with FUNCID.""" - self.tk.call('bind', self._w, sequence, '') - if funcid: + """Unbind for this widget the event SEQUENCE. + + If FUNCID is given, only unbind the function identified with FUNCID + and also delete the corresponding Tcl command. + + Otherwise destroy the current binding for SEQUENCE, leaving SEQUENCE + unbound. + """ + if funcid is None: + self.tk.call('bind', self._w, sequence, '') + else: + lines = self.tk.call('bind', self._w, sequence).split('\n') + prefix = f'if {{"[{funcid} ' + keep = '\n'.join(line for line in lines + if not line.startswith(prefix)) + if not keep.strip(): + keep = '' + self.tk.call('bind', self._w, sequence, keep) self.deletecommand(funcid) def bind_all(self, sequence=None, func=None, add=None): diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index efeabb7a92c627..5ca938a670831a 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -95,40 +95,47 @@ def _format_mapdict(mapdict, script=False): def _format_elemcreate(etype, script=False, *args, **kw): """Formats args and kw according to the given element factory etype.""" - spec = None + specs = () opts = () - if etype in ("image", "vsapi"): - if etype == "image": # define an element based on an image - # first arg should be the default image name - iname = args[0] - # next args, if any, are statespec/value pairs which is almost - # a mapdict, but we just need the value - imagespec = _join(_mapdict_values(args[1:])) - spec = "%s %s" % (iname, imagespec) - + if etype == "image": # define an element based on an image + # first arg should be the default image name + iname = args[0] + # next args, if any, are statespec/value pairs which is almost + # a mapdict, but we just need the value + imagespec = (iname, *_mapdict_values(args[1:])) + if script: + specs = (imagespec,) else: - # define an element whose visual appearance is drawn using the - # Microsoft Visual Styles API which is responsible for the - # themed styles on Windows XP and Vista. - # Availability: Tk 8.6, Windows XP and Vista. - class_name, part_id = args[:2] - statemap = _join(_mapdict_values(args[2:])) - spec = "%s %s %s" % (class_name, part_id, statemap) + specs = (_join(imagespec),) + opts = _format_optdict(kw, script) + if etype == "vsapi": + # define an element whose visual appearance is drawn using the + # Microsoft Visual Styles API which is responsible for the + # themed styles on Windows XP and Vista. + # Availability: Tk 8.6, Windows XP and Vista. + if len(args) < 3: + class_name, part_id = args + statemap = (((), 1),) + else: + class_name, part_id, statemap = args + specs = (class_name, part_id, tuple(_mapdict_values(statemap))) opts = _format_optdict(kw, script) elif etype == "from": # clone an element # it expects a themename and optionally an element to clone from, # otherwise it will clone {} (empty element) - spec = args[0] # theme name + specs = (args[0],) # theme name if len(args) > 1: # elementfrom specified opts = (_format_optvalue(args[1], script),) if script: - spec = '{%s}' % spec + specs = _join(specs) opts = ' '.join(opts) + return specs, opts + else: + return *specs, opts - return spec, opts def _format_layoutlist(layout, indent=0, indent_size=2): """Formats a layout list so we can pass the result to ttk::style @@ -214,10 +221,10 @@ def _script_from_settings(settings): elemargs = eopts[1:argc] elemkw = eopts[argc] if argc < len(eopts) and eopts[argc] else {} - spec, opts = _format_elemcreate(etype, True, *elemargs, **elemkw) + specs, eopts = _format_elemcreate(etype, True, *elemargs, **elemkw) script.append("ttk::style element create %s %s %s %s" % ( - name, etype, spec, opts)) + name, etype, specs, eopts)) return '\n'.join(script) @@ -434,9 +441,9 @@ def layout(self, style, layoutspec=None): def element_create(self, elementname, etype, *args, **kw): """Create a new element in the current theme of given etype.""" - spec, opts = _format_elemcreate(etype, False, *args, **kw) + *specs, opts = _format_elemcreate(etype, False, *args, **kw) self.tk.call(self._name, "element", "create", elementname, etype, - spec, *opts) + *specs, *opts) def element_names(self): diff --git a/Lib/traceback.py b/Lib/traceback.py index b25a7291f6be51..1cf008c7e9da97 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1,10 +1,13 @@ """Extract, format and print information about Python stack traces.""" +import os +import io import collections.abc import itertools import linecache import sys import textwrap +import warnings from contextlib import suppress __all__ = ['extract_stack', 'extract_tb', 'format_exception', @@ -18,6 +21,8 @@ # Formatting and printing lists of traceback lines. # +_COLORIZE = True + def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or extract_stack() as a formatted stack trace to the given file.""" @@ -109,7 +114,7 @@ def _parse_value_tb(exc, value, tb): def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - file=None, chain=True): + file=None, chain=True, **kwargs): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -120,17 +125,44 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ occurred with a caret on the next line indicating the approximate position of the error. """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - te.print(file=file, chain=chain) + te.print(file=file, chain=chain, colorize=colorize) BUILTIN_EXCEPTION_LIMIT = object() +def _can_colorize(): + if sys.platform == "win32": + try: + import nt + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + + if os.environ.get("PYTHON_COLORS") == "0": + return False + if os.environ.get("PYTHON_COLORS") == "1": + return True + if "NO_COLOR" in os.environ: + return False + if not _COLORIZE: + return False + if "FORCE_COLOR" in os.environ: + return True + if os.environ.get("TERM") == "dumb": + return False + try: + return os.isatty(sys.stderr.fileno()) + except io.UnsupportedOperation: + return sys.stderr.isatty() def _print_exception_bltin(exc, /): file = sys.stderr if sys.stderr is not None else sys.__stderr__ - return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file) + colorize = _can_colorize() + return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ @@ -171,13 +203,19 @@ def format_exception_only(exc, /, value=_sentinel, *, show_group=False): # -- not official API but folk probably use these two functions. -def _format_final_exc_line(etype, value, *, insert_final_newline=True): +def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize=False): valuestr = _safe_string(value, 'exception') end_char = "\n" if insert_final_newline else "" - if value is None or not valuestr: - line = f"{etype}{end_char}" + if colorize: + if value is None or not valuestr: + line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}{end_char}" + else: + line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}: {_ANSIColors.MAGENTA}{valuestr}{_ANSIColors.RESET}{end_char}" else: - line = f"{etype}: {valuestr}{end_char}" + if value is None or not valuestr: + line = f"{etype}{end_char}" + else: + line = f"{etype}: {valuestr}{end_char}" return line def _safe_string(value, what, func=str): @@ -273,7 +311,7 @@ class FrameSummary: """ __slots__ = ('filename', 'lineno', 'end_lineno', 'colno', 'end_colno', - 'name', '_line', 'locals') + 'name', '_lines', '_lines_dedented', 'locals') def __init__(self, filename, lineno, name, *, lookup_line=True, locals=None, line=None, @@ -289,15 +327,16 @@ def __init__(self, filename, lineno, name, *, lookup_line=True, """ self.filename = filename self.lineno = lineno + self.end_lineno = lineno if end_lineno is None else end_lineno + self.colno = colno + self.end_colno = end_colno self.name = name - self._line = line + self._lines = line + self._lines_dedented = None if lookup_line: self.line self.locals = {k: _safe_string(v, 'local', func=repr) for k, v in locals.items()} if locals else None - self.end_lineno = end_lineno - self.colno = colno - self.end_colno = end_colno def __eq__(self, other): if isinstance(other, FrameSummary): @@ -322,19 +361,39 @@ def __repr__(self): def __len__(self): return 4 + def _set_lines(self): + if ( + self._lines is None + and self.lineno is not None + and self.end_lineno is not None + ): + lines = [] + for lineno in range(self.lineno, self.end_lineno + 1): + # treat errors (empty string) and empty lines (newline) as the same + lines.append(linecache.getline(self.filename, lineno).rstrip()) + self._lines = "\n".join(lines) + "\n" + @property - def _original_line(self): + def _original_lines(self): # Returns the line as-is from the source, without modifying whitespace. - self.line - return self._line + self._set_lines() + return self._lines + + @property + def _dedented_lines(self): + # Returns _original_lines, but dedented + self._set_lines() + if self._lines_dedented is None and self._lines is not None: + self._lines_dedented = textwrap.dedent(self._lines) + return self._lines_dedented @property def line(self): - if self._line is None: - if self.lineno is None: - return None - self._line = linecache.getline(self.filename, self.lineno) - return self._line.strip() + self._set_lines() + if self._lines is None: + return None + # return only the first line, stripped + return self._lines.partition("\n")[0].strip() def walk_stack(f): @@ -384,6 +443,14 @@ def _get_code_position(code, instruction_index): _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. +class _ANSIColors: + RED = '\x1b[31m' + BOLD_RED = '\x1b[1;31m' + MAGENTA = '\x1b[35m' + BOLD_MAGENTA = '\x1b[1;35m' + GREY = '\x1b[90m' + RESET = '\x1b[0m' + class StackSummary(list): """A list of FrameSummary objects, representing a stack of frames.""" @@ -474,75 +541,193 @@ def from_list(klass, a_list): result.append(FrameSummary(filename, lineno, name, line=line)) return result - def format_frame_summary(self, frame_summary): + def format_frame_summary(self, frame_summary, **kwargs): """Format the lines for a single FrameSummary. Returns a string representing one frame involved in the stack. This gets called for every frame to be printed in the stack summary. """ + colorize = kwargs.get("colorize", False) row = [] filename = frame_summary.filename if frame_summary.filename.startswith("-"): filename = "" - row.append(' File "{}", line {}, in {}\n'.format( - filename, frame_summary.lineno, frame_summary.name)) - if frame_summary.line: - stripped_line = frame_summary.line.strip() - row.append(' {}\n'.format(stripped_line)) - - line = frame_summary._original_line - orig_line_len = len(line) - frame_line_len = len(frame_summary.line.lstrip()) - stripped_characters = orig_line_len - frame_line_len + if colorize: + row.append(' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( + _ANSIColors.MAGENTA, + filename, + _ANSIColors.RESET, + _ANSIColors.MAGENTA, + frame_summary.lineno, + _ANSIColors.RESET, + _ANSIColors.MAGENTA, + frame_summary.name, + _ANSIColors.RESET, + ) + ) + else: + row.append(' File "{}", line {}, in {}\n'.format( + filename, frame_summary.lineno, frame_summary.name)) + if frame_summary._dedented_lines and frame_summary._dedented_lines.strip(): if ( - frame_summary.colno is not None - and frame_summary.end_colno is not None + frame_summary.colno is None or + frame_summary.end_colno is None ): - start_offset = _byte_offset_to_character_offset( - line, frame_summary.colno) - end_offset = _byte_offset_to_character_offset( - line, frame_summary.end_colno) - code_segment = line[start_offset:end_offset] + # only output first line if column information is missing + row.append(textwrap.indent(frame_summary.line, ' ') + "\n") + else: + # get first and last line + all_lines_original = frame_summary._original_lines.splitlines() + first_line = all_lines_original[0] + # assume all_lines_original has enough lines (since we constructed it) + last_line = all_lines_original[frame_summary.end_lineno - frame_summary.lineno] + + # character index of the start/end of the instruction + start_offset = _byte_offset_to_character_offset(first_line, frame_summary.colno) + end_offset = _byte_offset_to_character_offset(last_line, frame_summary.end_colno) + + all_lines = frame_summary._dedented_lines.splitlines()[ + :frame_summary.end_lineno - frame_summary.lineno + 1 + ] + + # adjust start/end offset based on dedent + dedent_characters = len(first_line) - len(all_lines[0]) + start_offset = max(0, start_offset - dedent_characters) + end_offset = max(0, end_offset - dedent_characters) + # When showing this on a terminal, some of the non-ASCII characters + # might be rendered as double-width characters, so we need to take + # that into account when calculating the length of the line. + dp_start_offset = _display_width(all_lines[0], offset=start_offset) + dp_end_offset = _display_width(all_lines[-1], offset=end_offset) + + # get exact code segment corresponding to the instruction + segment = "\n".join(all_lines) + segment = segment[start_offset:len(segment) - (len(all_lines[-1]) - end_offset)] + + # attempt to parse for anchors anchors = None - if frame_summary.lineno == frame_summary.end_lineno: - with suppress(Exception): - anchors = _extract_caret_anchors_from_line_segment(code_segment) - else: - # Don't count the newline since the anchors only need to - # go up until the last character of the line. - end_offset = len(line.rstrip()) - - # show indicators if primary char doesn't span the frame line - if end_offset - start_offset < len(stripped_line) or ( - anchors and anchors.right_start_offset - anchors.left_end_offset > 0): - # When showing this on a terminal, some of the non-ASCII characters - # might be rendered as double-width characters, so we need to take - # that into account when calculating the length of the line. - dp_start_offset = _display_width(line, start_offset) + 1 - dp_end_offset = _display_width(line, end_offset) + 1 - - row.append(' ') - row.append(' ' * (dp_start_offset - stripped_characters)) - - if anchors: - dp_left_end_offset = _display_width(code_segment, anchors.left_end_offset) - dp_right_start_offset = _display_width(code_segment, anchors.right_start_offset) - row.append(anchors.primary_char * dp_left_end_offset) - row.append(anchors.secondary_char * (dp_right_start_offset - dp_left_end_offset)) - row.append(anchors.primary_char * (dp_end_offset - dp_start_offset - dp_right_start_offset)) - else: - row.append('^' * (dp_end_offset - dp_start_offset)) + with suppress(Exception): + anchors = _extract_caret_anchors_from_line_segment(segment) + + # only use carets if there are anchors or the carets do not span all lines + show_carets = False + if anchors or all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): + show_carets = True + + result = [] + + # only display first line, last line, and lines around anchor start/end + significant_lines = {0, len(all_lines) - 1} + + anchors_left_end_offset = 0 + anchors_right_start_offset = 0 + primary_char = "^" + secondary_char = "^" + if anchors: + anchors_left_end_offset = anchors.left_end_offset + anchors_right_start_offset = anchors.right_start_offset + # computed anchor positions do not take start_offset into account, + # so account for it here + if anchors.left_end_lineno == 0: + anchors_left_end_offset += start_offset + if anchors.right_start_lineno == 0: + anchors_right_start_offset += start_offset + + # account for display width + anchors_left_end_offset = _display_width( + all_lines[anchors.left_end_lineno], offset=anchors_left_end_offset + ) + anchors_right_start_offset = _display_width( + all_lines[anchors.right_start_lineno], offset=anchors_right_start_offset + ) - row.append('\n') + primary_char = anchors.primary_char + secondary_char = anchors.secondary_char + significant_lines.update( + range(anchors.left_end_lineno - 1, anchors.left_end_lineno + 2) + ) + significant_lines.update( + range(anchors.right_start_lineno - 1, anchors.right_start_lineno + 2) + ) + # remove bad line numbers + significant_lines.discard(-1) + significant_lines.discard(len(all_lines)) + + def output_line(lineno): + """output all_lines[lineno] along with carets""" + result.append(all_lines[lineno] + "\n") + if not show_carets: + return + num_spaces = len(all_lines[lineno]) - len(all_lines[lineno].lstrip()) + carets = [] + num_carets = dp_end_offset if lineno == len(all_lines) - 1 else _display_width(all_lines[lineno]) + # compute caret character for each position + for col in range(num_carets): + if col < num_spaces or (lineno == 0 and col < dp_start_offset): + # before first non-ws char of the line, or before start of instruction + carets.append(' ') + elif anchors and ( + lineno > anchors.left_end_lineno or + (lineno == anchors.left_end_lineno and col >= anchors_left_end_offset) + ) and ( + lineno < anchors.right_start_lineno or + (lineno == anchors.right_start_lineno and col < anchors_right_start_offset) + ): + # within anchors + carets.append(secondary_char) + else: + carets.append(primary_char) + if colorize: + # Replace the previous line with a red version of it only in the parts covered + # by the carets. + line = result[-1] + colorized_line_parts = [] + colorized_carets_parts = [] + + for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): + caret_group = list(group) + if color == "^": + colorized_line_parts.append(_ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) + colorized_carets_parts.append(_ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + elif color == "~": + colorized_line_parts.append(_ANSIColors.RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) + colorized_carets_parts.append(_ANSIColors.RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + else: + colorized_line_parts.append("".join(char for char, _ in caret_group)) + colorized_carets_parts.append("".join(caret for _, caret in caret_group)) + + colorized_line = "".join(colorized_line_parts) + colorized_carets = "".join(colorized_carets_parts) + result[-1] = colorized_line + result.append(colorized_carets + "\n") + else: + result.append("".join(carets) + "\n") + + # display significant lines + sig_lines_list = sorted(significant_lines) + for i, lineno in enumerate(sig_lines_list): + if i: + linediff = lineno - sig_lines_list[i - 1] + if linediff == 2: + # 1 line in between - just output it + output_line(lineno - 1) + elif linediff > 2: + # > 1 line in between - abbreviate + result.append(f"...<{linediff - 1} lines>...\n") + output_line(lineno) + + row.append( + textwrap.indent(textwrap.dedent("".join(result)), ' ', lambda line: True) + ) if frame_summary.locals: for name, value in sorted(frame_summary.locals.items()): row.append(' {name} = {value}\n'.format(name=name, value=value)) return ''.join(row) - def format(self): + def format(self, **kwargs): """Format the stack ready for printing. Returns a list of strings ready for printing. Each string in the @@ -554,13 +739,14 @@ def format(self): repetitions are shown, followed by a summary line stating the exact number of further repetitions. """ + colorize = kwargs.get("colorize", False) result = [] last_file = None last_line = None last_name = None count = 0 for frame_summary in self: - formatted_frame = self.format_frame_summary(frame_summary) + formatted_frame = self.format_frame_summary(frame_summary, colorize=colorize) if formatted_frame is None: continue if (last_file is None or last_file != frame_summary.filename or @@ -598,7 +784,9 @@ def _byte_offset_to_character_offset(str, offset): _Anchors = collections.namedtuple( "_Anchors", [ + "left_end_lineno", "left_end_offset", + "right_start_lineno", "right_start_offset", "primary_char", "secondary_char", @@ -607,59 +795,161 @@ def _byte_offset_to_character_offset(str, offset): ) def _extract_caret_anchors_from_line_segment(segment): + """ + Given source code `segment` corresponding to a FrameSummary, determine: + - for binary ops, the location of the binary op + - for indexing and function calls, the location of the brackets. + `segment` is expected to be a valid Python expression. + """ import ast try: - tree = ast.parse(segment) + # Without parentheses, `segment` is parsed as a statement. + # Binary ops, subscripts, and calls are expressions, so + # we can wrap them with parentheses to parse them as + # (possibly multi-line) expressions. + # e.g. if we try to highlight the addition in + # x = ( + # a + + # b + # ) + # then we would ast.parse + # a + + # b + # which is not a valid statement because of the newline. + # Adding brackets makes it a valid expression. + # ( + # a + + # b + # ) + # Line locations will be different than the original, + # which is taken into account later on. + tree = ast.parse(f"(\n{segment}\n)") except SyntaxError: return None if len(tree.body) != 1: return None - normalize = lambda offset: _byte_offset_to_character_offset(segment, offset) + lines = segment.splitlines() + + def normalize(lineno, offset): + """Get character index given byte offset""" + return _byte_offset_to_character_offset(lines[lineno], offset) + + def next_valid_char(lineno, col): + """Gets the next valid character index in `lines`, if + the current location is not valid. Handles empty lines. + """ + while lineno < len(lines) and col >= len(lines[lineno]): + col = 0 + lineno += 1 + assert lineno < len(lines) and col < len(lines[lineno]) + return lineno, col + + def increment(lineno, col): + """Get the next valid character index in `lines`.""" + col += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def nextline(lineno, col): + """Get the next valid character at least on the next line""" + col = 0 + lineno += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def increment_until(lineno, col, stop): + """Get the next valid non-"\\#" character that satisfies the `stop` predicate""" + while True: + ch = lines[lineno][col] + if ch in "\\#": + lineno, col = nextline(lineno, col) + elif not stop(ch): + lineno, col = increment(lineno, col) + else: + break + return lineno, col + + def setup_positions(expr, force_valid=True): + """Get the lineno/col position of the end of `expr`. If `force_valid` is True, + forces the position to be a valid character (e.g. if the position is beyond the + end of the line, move to the next line) + """ + # -2 since end_lineno is 1-indexed and because we added an extra + # bracket + newline to `segment` when calling ast.parse + lineno = expr.end_lineno - 2 + col = normalize(lineno, expr.end_col_offset) + return next_valid_char(lineno, col) if force_valid else (lineno, col) + statement = tree.body[0] match statement: case ast.Expr(expr): match expr: case ast.BinOp(): - operator_start = normalize(expr.left.end_col_offset) - operator_end = normalize(expr.right.col_offset) - operator_str = segment[operator_start:operator_end] - operator_offset = len(operator_str) - len(operator_str.lstrip()) + # ast gives these locations for BinOp subexpressions + # ( left_expr ) + ( right_expr ) + # left^^^^^ right^^^^^ + lineno, col = setup_positions(expr.left) + + # First operator character is the first non-space/')' character + lineno, col = increment_until(lineno, col, lambda x: not x.isspace() and x != ')') - left_anchor = expr.left.end_col_offset + operator_offset - right_anchor = left_anchor + 1 + # binary op is 1 or 2 characters long, on the same line, + # before the right subexpression + right_col = col + 1 if ( - operator_offset + 1 < len(operator_str) - and not operator_str[operator_offset + 1].isspace() + right_col < len(lines[lineno]) + and ( + # operator char should not be in the right subexpression + expr.right.lineno - 2 > lineno or + right_col < normalize(expr.right.lineno - 2, expr.right.col_offset) + ) + and not (ch := lines[lineno][right_col]).isspace() + and ch not in "\\#" ): - right_anchor += 1 + right_col += 1 - while left_anchor < len(segment) and ((ch := segment[left_anchor]).isspace() or ch in ")#"): - left_anchor += 1 - right_anchor += 1 - return _Anchors(normalize(left_anchor), normalize(right_anchor)) + # right_col can be invalid since it is exclusive + return _Anchors(lineno, col, lineno, right_col) case ast.Subscript(): - left_anchor = normalize(expr.value.end_col_offset) - right_anchor = normalize(expr.slice.end_col_offset + 1) - while left_anchor < len(segment) and ((ch := segment[left_anchor]).isspace() or ch != "["): - left_anchor += 1 - while right_anchor < len(segment) and ((ch := segment[right_anchor]).isspace() or ch != "]"): - right_anchor += 1 - if right_anchor < len(segment): - right_anchor += 1 - return _Anchors(left_anchor, right_anchor) + # ast gives these locations for value and slice subexpressions + # ( value_expr ) [ slice_expr ] + # value^^^^^ slice^^^^^ + # subscript^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.value) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '[') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + case ast.Call(): + # ast gives these locations for function call expressions + # ( func_expr ) (args, kwargs) + # func^^^^^ + # call^^^^^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.func) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '(') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) return None _WIDE_CHAR_SPECIFIERS = "WF" -def _display_width(line, offset): +def _display_width(line, offset=None): """Calculate the extra amount of width space the given source code segment might take if it were to be displayed on a fixed width output device. Supports wide unicode characters and emojis.""" + if offset is None: + offset = len(line) + # Fast track for ASCII-only strings if line.isascii(): return offset @@ -719,7 +1009,8 @@ class TracebackException: - :attr:`__suppress_context__` The *__suppress_context__* value from the original exception. - :attr:`stack` A `StackSummary` representing the traceback. - - :attr:`exc_type` The class of the original traceback. + - :attr:`exc_type` (deprecated) The class of the original traceback. + - :attr:`exc_type_str` String display of exc_type - :attr:`filename` For syntax errors - the filename where the error occurred. - :attr:`lineno` For syntax errors - the linenumber where the error @@ -737,7 +1028,7 @@ class TracebackException: def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, - max_group_width=15, max_group_depth=10, _seen=None): + max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. @@ -754,12 +1045,23 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, _walk_tb_with_full_positions(exc_traceback), limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals) - self.exc_type = exc_type + + self._exc_type = exc_type if save_exc_type else None + # Capture now to permit freeing resources: only complication is in the # unofficial API _format_final_exc_line self._str = _safe_string(exc_value, 'exception') self.__notes__ = getattr(exc_value, '__notes__', None) + self._is_syntax_error = False + self._have_exc_type = exc_type is not None + if exc_type is not None: + self.exc_type_qualname = exc_type.__qualname__ + self.exc_type_module = exc_type.__module__ + else: + self.exc_type_qualname = None + self.exc_type_module = None + if exc_type and issubclass(exc_type, SyntaxError): # Handle SyntaxError's specially self.filename = exc_value.filename @@ -771,6 +1073,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, self.offset = exc_value.offset self.end_offset = exc_value.end_offset self.msg = exc_value.msg + self._is_syntax_error = True elif exc_type and issubclass(exc_type, ImportError) and \ getattr(exc_value, "name_from", None) is not None: wrong_name = getattr(exc_value, "name_from", None) @@ -869,6 +1172,24 @@ def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) + @property + def exc_type(self): + warnings.warn('Deprecated in 3.13. Use exc_type_str instead.', + DeprecationWarning, stacklevel=2) + return self._exc_type + + @property + def exc_type_str(self): + if not self._have_exc_type: + return None + stype = self.exc_type_qualname + smod = self.exc_type_module + if smod not in ("__main__", "builtins"): + if not isinstance(smod, str): + smod = "" + stype = smod + '.' + stype + return stype + def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" for frame in self.stack: @@ -882,7 +1203,7 @@ def __eq__(self, other): def __str__(self): return self._str - def format_exception_only(self, *, show_group=False, _depth=0): + def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): """Format the exception part of the traceback. The return value is a generator of strings, each ending in a newline. @@ -899,33 +1220,28 @@ def format_exception_only(self, *, show_group=False, _depth=0): :exc:`BaseExceptionGroup`, the nested exceptions are included as well, recursively, with indentation relative to their nesting depth. """ + colorize = kwargs.get("colorize", False) indent = 3 * _depth * ' ' - if self.exc_type is None: - yield indent + _format_final_exc_line(None, self._str) + if not self._have_exc_type: + yield indent + _format_final_exc_line(None, self._str, colorize=colorize) return - stype = self.exc_type.__qualname__ - smod = self.exc_type.__module__ - if smod not in ("__main__", "builtins"): - if not isinstance(smod, str): - smod = "" - stype = smod + '.' + stype - - if not issubclass(self.exc_type, SyntaxError): + stype = self.exc_type_str + if not self._is_syntax_error: if _depth > 0: # Nested exceptions needs correct handling of multiline messages. formatted = _format_final_exc_line( - stype, self._str, insert_final_newline=False, + stype, self._str, insert_final_newline=False, colorize=colorize ).split('\n') yield from [ indent + l + '\n' for l in formatted ] else: - yield _format_final_exc_line(stype, self._str) + yield _format_final_exc_line(stype, self._str, colorize=colorize) else: - yield from [indent + l for l in self._format_syntax_error(stype)] + yield from [indent + l for l in self._format_syntax_error(stype, colorize=colorize)] if ( isinstance(self.__notes__, collections.abc.Sequence) @@ -939,15 +1255,26 @@ def format_exception_only(self, *, show_group=False, _depth=0): if self.exceptions and show_group: for ex in self.exceptions: - yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1) + yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1, colorize=colorize) - def _format_syntax_error(self, stype): + def _format_syntax_error(self, stype, **kwargs): """Format SyntaxError exceptions (internal helper).""" # Show exactly where the problem was found. + colorize = kwargs.get("colorize", False) filename_suffix = '' if self.lineno is not None: - yield ' File "{}", line {}\n'.format( - self.filename or "", self.lineno) + if colorize: + yield ' File {}"{}"{}, line {}{}{}\n'.format( + _ANSIColors.MAGENTA, + self.filename or "", + _ANSIColors.RESET, + _ANSIColors.MAGENTA, + self.lineno, + _ANSIColors.RESET, + ) + else: + yield ' File "{}", line {}\n'.format( + self.filename or "", self.lineno) elif self.filename is not None: filename_suffix = ' ({})'.format(self.filename) @@ -959,9 +1286,9 @@ def _format_syntax_error(self, stype): rtext = text.rstrip('\n') ltext = rtext.lstrip(' \n\f') spaces = len(rtext) - len(ltext) - yield ' {}\n'.format(ltext) - - if self.offset is not None: + if self.offset is None: + yield ' {}\n'.format(ltext) + else: offset = self.offset end_offset = self.end_offset if self.end_offset not in {None, 0} else offset if self.text and offset > len(self.text): @@ -974,14 +1301,43 @@ def _format_syntax_error(self, stype): # Convert 1-based column offset to 0-based index into stripped text colno = offset - 1 - spaces end_colno = end_offset - 1 - spaces + caretspace = ' ' if colno >= 0: # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c if c.isspace() else ' ') for c in ltext[:colno]) - yield ' {}{}'.format("".join(caretspace), ('^' * (end_colno - colno) + "\n")) + start_color = end_color = "" + if colorize: + # colorize from colno to end_colno + ltext = ( + ltext[:colno] + + _ANSIColors.BOLD_RED + ltext[colno:end_colno] + _ANSIColors.RESET + + ltext[end_colno:] + ) + start_color = _ANSIColors.BOLD_RED + end_color = _ANSIColors.RESET + yield ' {}\n'.format(ltext) + yield ' {}{}{}{}\n'.format( + "".join(caretspace), + start_color, + ('^' * (end_colno - colno)), + end_color, + ) + else: + yield ' {}\n'.format(ltext) msg = self.msg or "" - yield "{}: {}{}\n".format(stype, msg, filename_suffix) + if colorize: + yield "{}{}{}: {}{}{}{}\n".format( + _ANSIColors.BOLD_MAGENTA, + stype, + _ANSIColors.RESET, + _ANSIColors.MAGENTA, + msg, + _ANSIColors.RESET, + filename_suffix) + else: + yield "{}: {}{}\n".format(stype, msg, filename_suffix) - def format(self, *, chain=True, _ctx=None): + def format(self, *, chain=True, _ctx=None, **kwargs): """Format the exception. If chain is not *True*, *__cause__* and *__context__* will not be formatted. @@ -993,7 +1349,7 @@ def format(self, *, chain=True, _ctx=None): The message indicating which exception occurred is always the last string in the output. """ - + colorize = kwargs.get("colorize", False) if _ctx is None: _ctx = _ExceptionPrintContext() @@ -1023,8 +1379,8 @@ def format(self, *, chain=True, _ctx=None): if exc.exceptions is None: if exc.stack: yield from _ctx.emit('Traceback (most recent call last):\n') - yield from _ctx.emit(exc.stack.format()) - yield from _ctx.emit(exc.format_exception_only()) + yield from _ctx.emit(exc.stack.format(colorize=colorize)) + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) elif _ctx.exception_group_depth > self.max_group_depth: # exception group, but depth exceeds limit yield from _ctx.emit( @@ -1039,9 +1395,9 @@ def format(self, *, chain=True, _ctx=None): yield from _ctx.emit( 'Exception Group Traceback (most recent call last):\n', margin_char = '+' if is_toplevel else None) - yield from _ctx.emit(exc.stack.format()) + yield from _ctx.emit(exc.stack.format(colorize=colorize)) - yield from _ctx.emit(exc.format_exception_only()) + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) num_excs = len(exc.exceptions) if num_excs <= self.max_group_width: n = num_excs @@ -1082,11 +1438,12 @@ def format(self, *, chain=True, _ctx=None): _ctx.exception_group_depth = 0 - def print(self, *, file=None, chain=True): + def print(self, *, file=None, chain=True, **kwargs): """Print the result of self.format(chain=chain) to 'file'.""" + colorize = kwargs.get("colorize", False) if file is None: file = sys.stderr - for line in self.format(chain=chain): + for line in self.format(chain=chain, colorize=colorize): print(line, file=file, end="") diff --git a/Lib/typing.py b/Lib/typing.py index 872aca82c4e779..d7d793539b35b1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -490,7 +490,7 @@ def __getitem__(self, parameters): return self._getitem(self, parameters) -class _LiteralSpecialForm(_SpecialForm, _root=True): +class _TypedCacheSpecialForm(_SpecialForm, _root=True): def __getitem__(self, parameters): if not isinstance(parameters, tuple): parameters = (parameters,) @@ -723,7 +723,7 @@ def Optional(self, parameters): arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] -@_LiteralSpecialForm +@_TypedCacheSpecialForm @_tp_cache(typed=True) def Literal(self, *parameters): """Special typing form to define literal types (a.k.a. value types). @@ -1782,6 +1782,31 @@ def _pickle_pskwargs(pskwargs): del _pickle_psargs, _pickle_pskwargs +# Preload these once, as globals, as a micro-optimisation. +# This makes a significant difference to the time it takes +# to do `isinstance()`/`issubclass()` checks +# against runtime-checkable protocols with only one callable member. +_abc_instancecheck = ABCMeta.__instancecheck__ +_abc_subclasscheck = ABCMeta.__subclasscheck__ + + +def _type_check_issubclass_arg_1(arg): + """Raise TypeError if `arg` is not an instance of `type` + in `issubclass(arg, )`. + + In most cases, this is verified by type.__subclasscheck__. + Checking it again unnecessarily would slow down issubclass() checks, + so, we don't perform this check unless we absolutely have to. + + For various error paths, however, + we want to ensure that *this* error message is shown to the user + where relevant, rather than a typing.py-specific error message. + """ + if not isinstance(arg, type): + # Same error message as for issubclass(1, int). + raise TypeError('issubclass() arg 1 must be a class') + + class _ProtocolMeta(ABCMeta): # This metaclass is somewhat unfortunate, # but is necessary for several reasons... @@ -1821,13 +1846,11 @@ def __subclasscheck__(cls, other): getattr(cls, '_is_protocol', False) and not _allow_reckless_class_checks() ): - if not isinstance(other, type): - # Same error message as for issubclass(1, int). - raise TypeError('issubclass() arg 1 must be a class') if ( not cls.__callable_proto_members_only__ and cls.__dict__.get("__subclasshook__") is _proto_hook ): + _type_check_issubclass_arg_1(other) non_method_attrs = sorted( attr for attr in cls.__protocol_attrs__ if not callable(getattr(cls, attr, None)) @@ -1837,11 +1860,12 @@ def __subclasscheck__(cls, other): f" Non-method members: {str(non_method_attrs)[1:-1]}." ) if not getattr(cls, '_is_runtime_protocol', False): + _type_check_issubclass_arg_1(other) raise TypeError( "Instance and class checks can only be used with " "@runtime_checkable protocols" ) - return super().__subclasscheck__(other) + return _abc_subclasscheck(cls, other) def __instancecheck__(cls, instance): # We need this method for situations where attributes are @@ -1850,7 +1874,7 @@ def __instancecheck__(cls, instance): return type.__instancecheck__(cls, instance) if not getattr(cls, "_is_protocol", False): # i.e., it's a concrete subclass of a protocol - return super().__instancecheck__(instance) + return _abc_instancecheck(cls, instance) if ( not getattr(cls, '_is_runtime_protocol', False) and @@ -1859,7 +1883,7 @@ def __instancecheck__(cls, instance): raise TypeError("Instance and class checks can only be used with" " @runtime_checkable protocols") - if super().__instancecheck__(instance): + if _abc_instancecheck(cls, instance): return True getattr_static = _lazy_load_getattr_static() @@ -2005,8 +2029,9 @@ def __mro_entries__(self, bases): return (self.__origin__,) -@_SpecialForm -def Annotated(self, params): +@_TypedCacheSpecialForm +@_tp_cache(typed=True) +def Annotated(self, *params): """Add context-specific metadata to a type. Example: Annotated[int, runtime_check.Unsigned] indicates to the @@ -2053,7 +2078,7 @@ def Annotated(self, params): where T1, T2 etc. are TypeVars, which would be invalid, because only one type should be passed to Annotated. """ - if not isinstance(params, tuple) or len(params) < 2: + if len(params) < 2: raise TypeError("Annotated[...] should be used " "with at least two arguments (a type and an " "annotation).") @@ -2743,11 +2768,26 @@ def __new__(cls, typename, bases, ns): class_getitem = _generic_class_getitem nm_tpl.__class_getitem__ = classmethod(class_getitem) # update from user namespace without overriding special namedtuple attributes - for key in ns: + for key, val in ns.items(): if key in _prohibited: raise AttributeError("Cannot overwrite NamedTuple attribute " + key) - elif key not in _special and key not in nm_tpl._fields: - setattr(nm_tpl, key, ns[key]) + elif key not in _special: + if key not in nm_tpl._fields: + setattr(nm_tpl, key, val) + try: + set_name = type(val).__set_name__ + except AttributeError: + pass + else: + try: + set_name(val, nm_tpl, key) + except BaseException as e: + e.add_note( + f"Error calling __set_name__ on {type(val).__name__!r} " + f"instance {key!r} in {typename!r}" + ) + raise + if Generic in bases: nm_tpl.__init_subclass__() return nm_tpl @@ -2869,8 +2909,14 @@ def __new__(cls, name, bases, ns, total=True): for base in bases: annotations.update(base.__dict__.get('__annotations__', {})) - required_keys.update(base.__dict__.get('__required_keys__', ())) - optional_keys.update(base.__dict__.get('__optional_keys__', ())) + + base_required = base.__dict__.get('__required_keys__', set()) + required_keys |= base_required + optional_keys -= base_required + + base_optional = base.__dict__.get('__optional_keys__', set()) + required_keys -= base_optional + optional_keys |= base_optional annotations.update(own_annotations) for annotation_key, annotation_type in own_annotations.items(): @@ -2882,14 +2928,23 @@ def __new__(cls, name, bases, ns, total=True): annotation_origin = get_origin(annotation_type) if annotation_origin is Required: - required_keys.add(annotation_key) + is_required = True elif annotation_origin is NotRequired: - optional_keys.add(annotation_key) - elif total: + is_required = False + else: + is_required = total + + if is_required: required_keys.add(annotation_key) + optional_keys.discard(annotation_key) else: optional_keys.add(annotation_key) + required_keys.discard(annotation_key) + assert required_keys.isdisjoint(optional_keys), ( + f"Required keys overlap with optional keys in {name}:" + f" {required_keys=}, {optional_keys=}" + ) tp_dict.__annotations__ = annotations tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) @@ -3246,7 +3301,7 @@ def __enter__(self) -> 'TextIO': def reveal_type[T](obj: T, /) -> T: - """Reveal the inferred type of a variable. + """Ask a static type checker to reveal the inferred type of an expression. When a static type checker encounters a call to ``reveal_type()``, it will emit the inferred type of the argument:: @@ -3258,7 +3313,7 @@ def reveal_type[T](obj: T, /) -> T: will produce output similar to 'Revealed type is "builtins.int"'. At runtime, the function prints the runtime type of the - argument and returns it unchanged. + argument and returns the argument unchanged. """ print(f"Runtime type is {type(obj).__name__!r}", file=sys.stderr) return obj diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index c6b46eea657a21..2adb3d70662b1a 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -2229,8 +2229,11 @@ def __get__(self, obj, _type=None): return self.create_mock() -_CODE_ATTRS = dir(CodeType) -_CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) +try: + _CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) + _CODE_ATTRS = dir(CodeType) +except ValueError: + _CODE_SIG = None class AsyncMockMixin(Base): @@ -2250,9 +2253,12 @@ def __init__(self, /, *args, **kwargs): self.__dict__['_mock_await_count'] = 0 self.__dict__['_mock_await_args'] = None self.__dict__['_mock_await_args_list'] = _CallList() - code_mock = NonCallableMock(spec_set=_CODE_ATTRS) - code_mock.__dict__["_spec_class"] = CodeType - code_mock.__dict__["_spec_signature"] = _CODE_SIG + if _CODE_SIG: + code_mock = NonCallableMock(spec_set=_CODE_ATTRS) + code_mock.__dict__["_spec_class"] = CodeType + code_mock.__dict__["_spec_signature"] = _CODE_SIG + else: + code_mock = NonCallableMock(spec_set=CodeType) code_mock.co_flags = ( inspect.CO_COROUTINE + inspect.CO_VARARGS diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 8398981ce53b9c..a4e0609045a9d5 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -14,12 +14,9 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting + # Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi + hash -r 2> /dev/null if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -39,14 +36,18 @@ deactivate () { deactivate nondestructive # on Windows, a path can contain colons and backslashes and has to be converted: -if [ "${OSTYPE:-}" = "cygwin" ] || [ "${OSTYPE:-}" = "msys" ] ; then - # transform D:\path\to\venv to /d/path/to/venv on MSYS - # and to /cygdrive/d/path/to/venv on Cygwin - export VIRTUAL_ENV=$(cygpath "__VENV_DIR__") -else - # use the path as-is - export VIRTUAL_ENV="__VENV_DIR__" -fi +case "$(uname)" in + CYGWIN*|MSYS*) + # transform D:\path\to\venv to /d/path/to/venv on MSYS + # and to /cygdrive/d/path/to/venv on Cygwin + VIRTUAL_ENV=$(cygpath "__VENV_DIR__") + export VIRTUAL_ENV + ;; + *) + # use the path as-is + export VIRTUAL_ENV="__VENV_DIR__" + ;; +esac _OLD_VIRTUAL_PATH="$PATH" PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" @@ -69,9 +70,6 @@ if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then export PS1 fi -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting +# Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi +hash -r 2> /dev/null diff --git a/Lib/warnings.py b/Lib/warnings.py index 32e58072b9cc33..b8ff078569d2ce 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -5,7 +5,7 @@ __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", - "resetwarnings", "catch_warnings"] + "resetwarnings", "catch_warnings", "deprecated"] def showwarning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" @@ -139,14 +139,18 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - assert action in ("error", "ignore", "always", "default", "module", - "once"), "invalid action: %r" % (action,) - assert isinstance(message, str), "message must be a string" - assert isinstance(category, type), "category must be a class" - assert issubclass(category, Warning), "category must be a Warning subclass" - assert isinstance(module, str), "module must be a string" - assert isinstance(lineno, int) and lineno >= 0, \ - "lineno must be an int >= 0" + if action not in {"error", "ignore", "always", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(message, str): + raise TypeError("message must be a string") + if not isinstance(category, type) or not issubclass(category, Warning): + raise TypeError("category must be a Warning subclass") + if not isinstance(module, str): + raise TypeError("module must be a string") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") if message or module: import re @@ -172,10 +176,12 @@ def simplefilter(action, category=Warning, lineno=0, append=False): 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - assert action in ("error", "ignore", "always", "default", "module", - "once"), "invalid action: %r" % (action,) - assert isinstance(lineno, int) and lineno >= 0, \ - "lineno must be an int >= 0" + if action not in {"error", "ignore", "always", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") _add_filter(action, None, category, None, lineno, append=append) def _add_filter(*item, append): @@ -508,6 +514,135 @@ def __exit__(self, *exc_info): self._module._showwarnmsg_impl = self._showwarnmsg_impl +class deprecated: + """Indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, the type checker + will generate a diagnostic on usage of the deprecated object. + + Usage: + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the ``@overload`` decorator for the attribute to + exist on the overload as returned by ``get_overloads()``. + + See PEP 702 for details. + + """ + def __init__( + self, + message: str, + /, + *, + category: type[Warning] | None = DeprecationWarning, + stacklevel: int = 1, + ) -> None: + if not isinstance(message, str): + raise TypeError( + f"Expected an object of type str for 'message', not {type(message).__name__!r}" + ) + self.message = message + self.category = category + self.stacklevel = stacklevel + + def __call__(self, arg, /): + # Make sure the inner functions created below don't + # retain a reference to self. + msg = self.message + category = self.category + stacklevel = self.stacklevel + if category is None: + arg.__deprecated__ = msg + return arg + elif isinstance(arg, type): + import functools + from types import MethodType + + original_new = arg.__new__ + + @functools.wraps(original_new) + def __new__(cls, *args, **kwargs): + if cls is arg: + warn(msg, category=category, stacklevel=stacklevel + 1) + if original_new is not object.__new__: + return original_new(cls, *args, **kwargs) + # Mirrors a similar check in object.__new__. + elif cls.__init__ is object.__init__ and (args or kwargs): + raise TypeError(f"{cls.__name__}() takes no arguments") + else: + return original_new(cls) + + arg.__new__ = staticmethod(__new__) + + original_init_subclass = arg.__init_subclass__ + # We need slightly different behavior if __init_subclass__ + # is a bound method (likely if it was implemented in Python) + if isinstance(original_init_subclass, MethodType): + original_init_subclass = original_init_subclass.__func__ + + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + + arg.__init_subclass__ = classmethod(__init_subclass__) + # Or otherwise, which likely means it's a builtin such as + # object's implementation of __init_subclass__. + else: + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + + arg.__init_subclass__ = __init_subclass__ + + arg.__deprecated__ = __new__.__deprecated__ = msg + __init_subclass__.__deprecated__ = msg + return arg + elif callable(arg): + import functools + + @functools.wraps(arg) + def wrapper(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return arg(*args, **kwargs) + + arg.__deprecated__ = wrapper.__deprecated__ = msg + return wrapper + else: + raise TypeError( + "@deprecated decorator with non-None category must be applied to " + f"a class or callable, not {arg!r}" + ) + + _DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 8b0628745c57fc..636e8ca459d109 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -495,7 +495,12 @@ def register_standard_browsers(): register("microsoft-edge", None, Edge("MicrosoftEdge.exe")) else: # Prefer X browsers if present - if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): + # + # NOTE: Do not check for X11 browser on macOS, + # XQuartz installation sets a DISPLAY environment variable and will + # autostart when someone tries to access the display. Mac users in + # general don't need an X11 browser. + if sys.platform != "darwin" and (os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")): try: cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) @@ -569,6 +574,7 @@ def __init__(self, name='default'): super().__init__(name) def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) if self.name == 'default': script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser else: diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 2b28a079dbaa95..fe629ed1cf2fc5 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -2227,12 +2227,79 @@ def _compile(file, optimize=-1): return (fname, archivename) +def main(args=None): + import argparse + + description = 'A simple command-line interface for zipfile module.' + parser = argparse.ArgumentParser(description=description) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-l', '--list', metavar='', + help='Show listing of a zipfile') + group.add_argument('-e', '--extract', nargs=2, + metavar=('', ''), + help='Extract zipfile into target dir') + group.add_argument('-c', '--create', nargs='+', + metavar=('', ''), + help='Create zipfile from sources') + group.add_argument('-t', '--test', metavar='', + help='Test if a zipfile is valid') + parser.add_argument('--metadata-encoding', metavar='', + help='Specify encoding of member names for -l, -e and -t') + args = parser.parse_args(args) + + encoding = args.metadata_encoding + + if args.test is not None: + src = args.test + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + badfile = zf.testzip() + if badfile: + print("The following enclosed file is corrupted: {!r}".format(badfile)) + print("Done testing") + + elif args.list is not None: + src = args.list + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.printdir() + + elif args.extract is not None: + src, curdir = args.extract + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.extractall(curdir) + + elif args.create is not None: + if encoding: + print("Non-conforming encodings not supported with -c.", + file=sys.stderr) + sys.exit(1) + + zip_name = args.create.pop(0) + files = args.create + + def addToZip(zf, path, zippath): + if os.path.isfile(path): + zf.write(path, zippath, ZIP_DEFLATED) + elif os.path.isdir(path): + if zippath: + zf.write(path, zippath) + for nm in sorted(os.listdir(path)): + addToZip(zf, + os.path.join(path, nm), os.path.join(zippath, nm)) + # else: ignore + + with ZipFile(zip_name, 'w') as zf: + for path in files: + zippath = os.path.basename(path) + if not zippath: + zippath = os.path.basename(os.path.dirname(path)) + if zippath in ('', os.curdir, os.pardir): + zippath = '' + addToZip(zf, path, zippath) + + from ._path import ( # noqa: E402 Path, # used privately for tests CompleteDirs, # noqa: F401 ) - -# used privately for tests -from .__main__ import main # noqa: F401, E402 diff --git a/Lib/zipfile/__main__.py b/Lib/zipfile/__main__.py index a9e5fb1b8d72c4..868d99efc3c4a3 100644 --- a/Lib/zipfile/__main__.py +++ b/Lib/zipfile/__main__.py @@ -1,77 +1,4 @@ -import sys -import os -from . import ZipFile, ZIP_DEFLATED - - -def main(args=None): - import argparse - - description = 'A simple command-line interface for zipfile module.' - parser = argparse.ArgumentParser(description=description) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-l', '--list', metavar='', - help='Show listing of a zipfile') - group.add_argument('-e', '--extract', nargs=2, - metavar=('', ''), - help='Extract zipfile into target dir') - group.add_argument('-c', '--create', nargs='+', - metavar=('', ''), - help='Create zipfile from sources') - group.add_argument('-t', '--test', metavar='', - help='Test if a zipfile is valid') - parser.add_argument('--metadata-encoding', metavar='', - help='Specify encoding of member names for -l, -e and -t') - args = parser.parse_args(args) - - encoding = args.metadata_encoding - - if args.test is not None: - src = args.test - with ZipFile(src, 'r', metadata_encoding=encoding) as zf: - badfile = zf.testzip() - if badfile: - print("The following enclosed file is corrupted: {!r}".format(badfile)) - print("Done testing") - - elif args.list is not None: - src = args.list - with ZipFile(src, 'r', metadata_encoding=encoding) as zf: - zf.printdir() - - elif args.extract is not None: - src, curdir = args.extract - with ZipFile(src, 'r', metadata_encoding=encoding) as zf: - zf.extractall(curdir) - - elif args.create is not None: - if encoding: - print("Non-conforming encodings not supported with -c.", - file=sys.stderr) - sys.exit(1) - - zip_name = args.create.pop(0) - files = args.create - - def addToZip(zf, path, zippath): - if os.path.isfile(path): - zf.write(path, zippath, ZIP_DEFLATED) - elif os.path.isdir(path): - if zippath: - zf.write(path, zippath) - for nm in sorted(os.listdir(path)): - addToZip(zf, - os.path.join(path, nm), os.path.join(zippath, nm)) - # else: ignore - - with ZipFile(zip_name, 'w') as zf: - for path in files: - zippath = os.path.basename(path) - if not zippath: - zippath = os.path.basename(os.path.dirname(path)) - if zippath in ('', os.curdir, os.pardir): - zippath = '' - addToZip(zf, path, zippath) - +from . import main if __name__ == "__main__": main() diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 5b9f614f02f7af..823a82ee830465 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -352,7 +352,7 @@ def _read_directory(archive): with fp: # GH-87235: On macOS all file descriptors for /dev/fd/N share the same - # file offset, reset the file offset after scanning the zipfile diretory + # file offset, reset the file offset after scanning the zipfile directory # to not cause problems when some runs 'python3 /dev/fd/9 9%version% NSHighResolutionCapable + CFBundleAllowMixedLocalizations + diff --git a/Mac/Resources/app/Info.plist.in b/Mac/Resources/app/Info.plist.in index 4ec828ff176e7b..8362b19b361b62 100644 --- a/Mac/Resources/app/Info.plist.in +++ b/Mac/Resources/app/Info.plist.in @@ -58,5 +58,7 @@ (c) 2001-2023 Python Software Foundation. NSHighResolutionCapable + CFBundleAllowMixedLocalizations + diff --git a/Mac/Resources/framework/Info.plist.in b/Mac/Resources/framework/Info.plist.in index e131c205ef0b28..238441ce2c76c7 100644 --- a/Mac/Resources/framework/Info.plist.in +++ b/Mac/Resources/framework/Info.plist.in @@ -24,5 +24,7 @@ ???? CFBundleVersion %VERSION% + CFBundleAllowMixedLocalizations + diff --git a/Makefile.pre.in b/Makefile.pre.in index 3d766425abba34..6a64547e97d266 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -657,11 +657,13 @@ check-clean-src: @if test -n "$(VPATH)" -a \( \ -f "$(srcdir)/$(BUILDPYTHON)" \ -o -f "$(srcdir)/Programs/python.o" \ - -o -f "$(srcdir)\Python/frozen_modules/importlib._bootstrap.h" \ + -o -f "$(srcdir)/Python/frozen_modules/importlib._bootstrap.h" \ \); then \ echo "Error: The source directory ($(srcdir)) is not clean" ; \ echo "Building Python out of the source tree (in $(abs_builddir)) requires a clean source tree ($(abs_srcdir))" ; \ - echo "Try to run: make -C \"$(srcdir)\" clean" ; \ + echo "Build artifacts such as .o files, executables, and Python/frozen_modules/*.h must not exist within $(srcdir)." ; \ + echo "Try to run:" ; \ + echo " (cd \"$(srcdir)\" && make clean || git clean -fdx -e Doc/venv)" ; \ exit 1; \ fi @@ -1359,7 +1361,7 @@ regen-unicodedata: regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects + regen-test-levenshtein regen-global-objects regen-sbom @echo @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " @echo "make regen-configure and make regen-unicodedata should be run manually" @@ -1562,7 +1564,7 @@ Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h Objects/obmalloc.o: $(srcdir)/Objects/mimalloc/alloc.c \ - $(srcdir)/Objects/mimalloc/alloc-aligned.c \ + $(srcdir)/Objects/mimalloc/alloc-aligned.c \ $(srcdir)/Objects/mimalloc/alloc-posix.c \ $(srcdir)/Objects/mimalloc/arena.c \ $(srcdir)/Objects/mimalloc/bitmap.c \ @@ -1575,7 +1577,10 @@ Objects/obmalloc.o: $(srcdir)/Objects/mimalloc/alloc.c \ $(srcdir)/Objects/mimalloc/segment.c \ $(srcdir)/Objects/mimalloc/segment-map.c \ $(srcdir)/Objects/mimalloc/stats.c \ - $(srcdir)/Objects/mimalloc/prim/prim.c + $(srcdir)/Objects/mimalloc/prim/prim.c \ + $(srcdir)/Objects/mimalloc/prim/osx/prim.c \ + $(srcdir)/Objects/mimalloc/prim/unix/prim.c \ + $(srcdir)/Objects/mimalloc/prim/wasi/prim.c Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c @@ -1583,24 +1588,29 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c regen-cases: # Regenerate various files from Python/bytecodes.c # Pass CASESFLAG=-l to insert #line directives in the output - PYTHONPATH=$(srcdir)/Tools/cases_generator \ - $(PYTHON_FOR_REGEN) \ - $(srcdir)/Tools/cases_generator/generate_cases.py \ - $(CASESFLAG) \ - -o $(srcdir)/Python/generated_cases.c.h.new \ - -n $(srcdir)/Include/opcode_ids.h.new \ - -t $(srcdir)/Python/opcode_targets.h.new \ - -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \ - -e $(srcdir)/Python/executor_cases.c.h.new \ - -p $(srcdir)/Lib/_opcode_metadata.py.new \ - -a $(srcdir)/Python/abstract_interp_cases.c.h.new \ - $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_id_generator.py \ + -o $(srcdir)/Include/opcode_ids.h.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/target_generator.py \ + -o $(srcdir)/Python/opcode_targets.h.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/uop_id_generator.py \ + -o $(srcdir)/Include/internal/pycore_uop_ids.h.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/py_metadata_generator.py \ + -o $(srcdir)/Lib/_opcode_metadata.py.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_generator.py \ + -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ + -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ + -o $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/uop_metadata_generator.py -o \ + $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new - $(UPDATE_FILE) $(srcdir)/Python/abstract_interp_cases.c.h $(srcdir)/Python/abstract_interp_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -1785,7 +1795,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/warnings.h \ $(srcdir)/Include/cpython/weakrefobject.h \ \ - @MIMALLOC_HEADERS@ \ + $(MIMALLOC_HEADERS) \ \ $(srcdir)/Include/internal/pycore_abstract.h \ $(srcdir)/Include/internal/pycore_asdl.h \ @@ -1873,6 +1883,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ $(srcdir)/Include/internal/pycore_tracemalloc.h \ + $(srcdir)/Include/internal/pycore_tstate.h \ $(srcdir)/Include/internal/pycore_tuple.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_typevarobject.h \ @@ -1881,6 +1892,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_unicodeobject.h \ $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ $(srcdir)/Include/internal/pycore_uops.h \ + $(srcdir)/Include/internal/pycore_uop_metadata.h \ $(srcdir)/Include/internal/pycore_warnings.h \ $(srcdir)/Include/internal/pycore_weakref.h \ $(DTRACE_HEADERS) \ @@ -2161,6 +2173,7 @@ LIBSUBDIRS= asyncio \ json \ logging \ multiprocessing multiprocessing/dummy \ + pathlib \ pydoc_data \ re \ site-packages \ @@ -2194,6 +2207,9 @@ TESTSUBDIRS= idlelib/idle_test \ test/leakers \ test/libregrtest \ test/mathdata \ + test/regrtestdata \ + test/regrtestdata/import_from_tests \ + test/regrtestdata/import_from_tests/test_regrtest_b \ test/subprocessdata \ test/support \ test/support/_hypothesis_stubs \ @@ -2261,6 +2277,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/source \ test/test_json \ test/test_module \ + test/test_pathlib \ test/test_peg_generator \ test/test_sqlite3 \ test/test_tkinter \ @@ -2647,6 +2664,10 @@ autoconf: regen-configure: $(srcdir)/Tools/build/regen-configure.sh +.PHONY: regen-sbom +regen-sbom: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_sbom.py + # Create a tags file for vi tags:: ctags -w $(srcdir)/Include/*.h $(srcdir)/Include/cpython/*.h $(srcdir)/Include/internal/*.h @@ -2732,6 +2753,7 @@ clobber: clean -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) -rm -f python-config.py python-config + -rm -rf cross-build # Make things extra clean, before making a distribution: # remove all generated files, even Makefile[.pre] diff --git a/Misc/ACKS b/Misc/ACKS index 5fe3a177a26292..ab1255be2d58fa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -31,6 +31,7 @@ Farhan Ahmad Matthew Ahrens Nir Aides Akira +Ege Akman Yaniv Aknin Jyrki Alakuijala Tatiana Al-Chueyr @@ -788,6 +789,7 @@ Thomas Holmes Craig Holmquist Philip Homburg Naofumi Honda +Constantin Hong Weipeng Hong Jeffrey Honig Rob Hooft @@ -1459,6 +1461,7 @@ Paul Prescod Donovan Preston Eric Price Paul Price +Matt Prodani Iuliia Proskurnia Dorian Pula Jyrki Pulliainen @@ -1571,6 +1574,7 @@ Liam Routt Todd Rovito Craig Rowland Clinton Roy +Ujan RoyBandyopadhyay Paul Rubin Sam Ruby Demur Rumed diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index a9f25b482508ba..3186de75efd9c5 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -605,8 +605,8 @@ Opt out serialization/deserialization for _random.Random .. nonce: jxJ4yn .. section: Core and Builtins -Rename `PyPegen*` functions to `PyParser*`, so that we can remove the old -set of `PyParser*` functions that were using the old parser, but keep +Rename ``PyPegen*`` functions to ``PyParser*``, so that we can remove the old +set of ``PyParser*`` functions that were using the old parser, but keep everything backwards-compatible. .. @@ -2527,7 +2527,7 @@ in Python 3.4 and removed in Python 3.5. .. nonce: BE7zbu .. section: Library -Fix `cgi.parse_multipart` without content_length. Patch by Roger Duran +Fix ``cgi.parse_multipart`` without content_length. Patch by Roger Duran .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 26c44b6c1af0ed..63abcbd5a6499e 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -819,7 +819,7 @@ always available when needed. Patch by Mark Shannon. .. nonce: qKnSqV .. section: Core and Builtins -The threading debug (:envvar:`PYTHONTHREADDEBUG` environment variable) is +The threading debug (:envvar:`!PYTHONTHREADDEBUG` environment variable) is deprecated in Python 3.10 and will be removed in Python 3.12. This feature requires a debug build of Python. Patch by Victor Stinner. @@ -1642,9 +1642,9 @@ interval specified with nanosecond precision. .. nonce: UptGAn .. section: Library -Remove from the :mod:`configparser` module: the :class:`SafeConfigParser` -class, the :attr:`filename` property of the :class:`ParsingError` class, the -:meth:`readfp` method of the :class:`ConfigParser` class, deprecated since +Remove from the :mod:`configparser` module: the :class:`!SafeConfigParser` +class, the :attr:`!filename` property of the :class:`~configparser.ParsingError` class, the +:meth:`!readfp` method of the :class:`~configparser.ConfigParser` class, deprecated since Python 3.2. Patch by Hugo van Kemenade. @@ -2808,7 +2808,7 @@ behaves differently than the similar implementation in :mod:`sysconfig`. .. nonce: 3hmkWw .. section: Library -:class:`smtpd.MailmanProxy` is now removed as it is unusable without an +:class:`!smtpd.MailmanProxy` is now removed as it is unusable without an external module, ``mailman``. Patch by Donghee Na. .. @@ -3483,9 +3483,9 @@ Improved reprs of :mod:`threading` synchronization objects: Deprecated the following :mod:`unittest` functions, scheduled for removal in Python 3.13: -* :func:`~!unittest.findTestCases` -* :func:`~!unittest.makeSuite` -* :func:`~!unittest.getTestCaseNames` +* :func:`!findTestCases` +* :func:`!makeSuite` +* :func:`!getTestCaseNames` Use :class:`~unittest.TestLoader` methods instead: diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 52055b3fafd485..974d025c631a45 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -941,7 +941,7 @@ uvloop library. Make the :class:`configparser.ConfigParser` constructor raise :exc:`TypeError` if the ``interpolation`` parameter is not of type -:class:`configparser.Interpolation` +:class:`!configparser.Interpolation` .. diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst index 6e41f9cbd933b5..76699632db223a 100644 --- a/Misc/NEWS.d/3.11.0a7.rst +++ b/Misc/NEWS.d/3.11.0a7.rst @@ -717,7 +717,7 @@ Fix :class:`asyncio.Semaphore` re-aquiring FIFO order. .. nonce: uaEDcI .. section: Library -The :mod:`asynchat`, :mod:`asyncore` and :mod:`smtpd` modules have been +The :mod:`!asynchat`, :mod:`!asyncore` and :mod:`!smtpd` modules have been deprecated since at least Python 3.6. Their documentation and deprecation warnings and have now been updated to note they will removed in Python 3.12 (:pep:`594`). @@ -1038,8 +1038,8 @@ Add optional parameter *dir_fd* in :func:`shutil.rmtree`. .. nonce: AixHW7 .. section: Library -:meth:`~!unittest.TestProgram.usageExit` is marked deprecated, to be removed -in 3.13. +:meth:`!unittest.TestProgram.usageExit` is marked as deprecated, +to be removed in Python 3.13. .. @@ -1324,7 +1324,7 @@ extensions. .. section: Tests A test case for :func:`os.sendfile` is converted from deprecated -:mod:`asyncore` (see :pep:`594`) to :mod:`asyncio`. Patch by Oleg Iarygin. +:mod:`!asyncore` (see :pep:`594`) to :mod:`asyncio`. Patch by Oleg Iarygin. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 633738de92bef7..81ef69093005e8 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -1913,7 +1913,7 @@ Stinner. .. nonce: Uxc9al .. section: Library -Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` +Allow :mod:`venv` to pass along :envvar:`!PYTHON*` variables to ``ensurepip`` and ``pip`` when they do not impact path resolution .. @@ -3617,7 +3617,7 @@ allow access to handlers by name. .. nonce: uw6x5z .. section: Library -The :mod:`smtpd` module was removed per the schedule in :pep:`594`. +The :mod:`!smtpd` module was removed per the schedule in :pep:`594`. .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 1a04ed473f329d..dbc743abe8a767 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -695,7 +695,7 @@ Make sure ``patch.dict()`` can be applied on async functions. .. nonce: jUpzF3 .. section: Library -Remove modules :mod:`asyncore` and :mod:`asynchat`, which were deprecated by +Remove modules :mod:`!asyncore` and :mod:`!asynchat`, which were deprecated by :pep:`594`. .. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index 75246f3f13503e..ce2814bbe2e5ab 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -147,8 +147,8 @@ clinic. .. nonce: yRWQ1y .. section: Core and Builtins -Improve the output of ``co_lines`` by emitting only one entry for each line -range. +Improve the output of :meth:`codeobject.co_lines` by emitting only one entry +for each line range. .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 0944dfd0e90ab9..007a6ad4ffd4d4 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -880,7 +880,7 @@ Update the ``repr`` of :class:`typing.Unpack` according to :pep:`692`. .. section: Library Make :mod:`dis` display the names of the args for -:opcode:`CALL_INTRINSIC_*`. +:opcode:`!CALL_INTRINSIC_*`. .. diff --git a/Misc/NEWS.d/3.6.0a2.rst b/Misc/NEWS.d/3.6.0a2.rst index 1b336d7bc5137a..05b3d9f0463c1c 100644 --- a/Misc/NEWS.d/3.6.0a2.rst +++ b/Misc/NEWS.d/3.6.0a2.rst @@ -603,7 +603,7 @@ configuring text widget colors to a new function. .. nonce: RbyFuV .. section: IDLE -Rename many `idlelib/*.py` and `idle_test/test_*.py` files. Edit files to +Rename many ``idlelib/*.py`` and ``idle_test/test_*.py`` files. Edit files to replace old names with new names when the old name referred to the module rather than the class it contained. See the issue and IDLE section in What's New in 3.6 for more. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 2b9dbd5d63a87e..b56cda86f11faa 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -2006,8 +2006,8 @@ Improved support of custom data descriptors in :func:`help` and .. nonce: V4kNN3 .. section: Library -The `crypt` module now internally uses the `crypt_r()` library function -instead of `crypt()` when available. +The ``crypt`` module now internally uses the ``crypt_r()`` library function +instead of ``crypt()`` when available. .. @@ -5044,8 +5044,8 @@ functionality. .. nonce: C_K-J9 .. section: Library -`ConfigParser.items()` was fixed so that key-value pairs passed in via -`vars` are not included in the resulting output. +``ConfigParser.items()`` was fixed so that key-value pairs passed in via +:func:`vars` are not included in the resulting output. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 7e8bfa5c4364a9..3097245b74a511 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -255,7 +255,7 @@ all tags in a namespace. Patch by Stefan Behnel. .. nonce: Lpm-SI .. section: Library -`pathlib.path.link_to()` is now implemented. It creates a hard link pointing +``pathlib.path.link_to()`` is now implemented. It creates a hard link pointing to a path. .. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 9818c17705074b..b365e5fbfb0dbe 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -2069,7 +2069,7 @@ Restores instantiation of Windows IOCP event loops from the non-main thread. .. section: Library Add default implementation of the :meth:`ast.NodeVisitor.visit_Constant` -method which emits a deprecation warning and calls corresponding methody +method which emits a deprecation warning and calls corresponding methods ``visit_Num()``, ``visit_Str()``, etc. .. @@ -2534,7 +2534,7 @@ object when `self._spec_signature` exists. Patch by Elizabeth Uselton .. nonce: iXGuoi .. section: Library -Make `from tkinter import *` import only the expected objects. +Make ``from tkinter import *`` import only the expected objects. .. @@ -3117,9 +3117,9 @@ Ensure cookies with ``expires`` attribute are handled in .. section: Library Fix an unintended ValueError from :func:`subprocess.run` when checking for -conflicting `input` and `stdin` or `capture_output` and `stdout` or `stderr` -args when they were explicitly provided but with `None` values within a -passed in `**kwargs` dict rather than as passed directly by name. Patch +conflicting *input* and *stdin* or *capture_output* and *stdout* or *stderr* +args when they were explicitly provided but with ``None`` values within a +passed in ``**kwargs`` dict rather than as passed directly by name. Patch contributed by Rémi Lapeyre. .. @@ -3546,7 +3546,7 @@ Patch by Stein Karlsen. .. nonce: XaJDei .. section: Library -lib2to3 now recognizes expressions after ``*`` and `**` like in ``f(*[] or +lib2to3 now recognizes expressions after ``*`` and ``**`` like in ``f(*[] or [])``. .. diff --git a/Misc/NEWS.d/3.9.0a3.rst b/Misc/NEWS.d/3.9.0a3.rst index 8a94848427382b..bc7f4f9c5d39c1 100644 --- a/Misc/NEWS.d/3.9.0a3.rst +++ b/Misc/NEWS.d/3.9.0a3.rst @@ -454,7 +454,7 @@ resilients to inaccessible sys.path entries (importlib_metadata v1.4.0). .. nonce: _S5VjC .. section: Library -:class:`~!nntplib.NNTP` and :class:`~!nntplib.NNTP_SSL` now raise a +:class:`!NNTP` and :class:`!NNTP_SSL` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of a non-blocking socket. Patch by Donghee Na. @@ -498,7 +498,7 @@ prevent the creation of a non-blocking socket. Patch by Donghee Na. .. section: Library Updated the Gmane domain from news.gmane.org to news.gmane.io which is used -for examples of :class:`~!nntplib.NNTP` news reader server and nntplib tests. +for examples of :class:`!NNTP` news reader server and nntplib tests. .. diff --git a/Misc/NEWS.d/next/Build/2020-05-01-23-44-31.bpo-11102.Fw9zeS.rst b/Misc/NEWS.d/next/Build/2020-05-01-23-44-31.bpo-11102.Fw9zeS.rst new file mode 100644 index 00000000000000..6477538edf5550 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2020-05-01-23-44-31.bpo-11102.Fw9zeS.rst @@ -0,0 +1,2 @@ +The :func:`os.major`, :func:`os.makedev`, and :func:`os.minor` functions are +now available on HP-UX v3. diff --git a/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst b/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst new file mode 100644 index 00000000000000..e2240b7c656a2f --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst @@ -0,0 +1 @@ +Introduce ``Tools/wasm/wasi.py`` to simplify doing a WASI build. diff --git a/Misc/NEWS.d/next/Build/2023-12-08-11-33-37.gh-issue-112867.ZzDfXQ.rst b/Misc/NEWS.d/next/Build/2023-12-08-11-33-37.gh-issue-112867.ZzDfXQ.rst new file mode 100644 index 00000000000000..a36814854882bb --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-12-08-11-33-37.gh-issue-112867.ZzDfXQ.rst @@ -0,0 +1 @@ +Fix the build for the case that WITH_PYMALLOC_RADIX_TREE=0 set. diff --git a/Misc/NEWS.d/next/Build/2023-12-17-18-23-02.gh-issue-112536.8lr3Ep.rst b/Misc/NEWS.d/next/Build/2023-12-17-18-23-02.gh-issue-112536.8lr3Ep.rst new file mode 100644 index 00000000000000..a136eb47584993 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-12-17-18-23-02.gh-issue-112536.8lr3Ep.rst @@ -0,0 +1 @@ +Add support for thread sanitizer (TSAN) diff --git a/Misc/NEWS.d/next/Build/2023-12-21-05-35-06.gh-issue-112305.VfqQPx.rst b/Misc/NEWS.d/next/Build/2023-12-21-05-35-06.gh-issue-112305.VfqQPx.rst new file mode 100644 index 00000000000000..2df3207f4e6f6c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-12-21-05-35-06.gh-issue-112305.VfqQPx.rst @@ -0,0 +1,3 @@ +Fixed the ``check-clean-src`` step performed on out of tree builds to detect +errant ``$(srcdir)/Python/frozen_modules/*.h`` files and recommend +appropriate source tree cleanup steps to get a working build again. diff --git a/Misc/NEWS.d/next/Build/2023-12-23-09-35-48.gh-issue-113258.GlsAyH.rst b/Misc/NEWS.d/next/Build/2023-12-23-09-35-48.gh-issue-113258.GlsAyH.rst new file mode 100644 index 00000000000000..e7256ea423b3e0 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-12-23-09-35-48.gh-issue-113258.GlsAyH.rst @@ -0,0 +1,2 @@ +Changed the Windows build to write out generated frozen modules into the +build tree instead of the source tree. diff --git a/Misc/NEWS.d/next/C API/2023-06-21-11-53-09.gh-issue-65210.PhFRBJ.rst b/Misc/NEWS.d/next/C API/2023-06-21-11-53-09.gh-issue-65210.PhFRBJ.rst new file mode 100644 index 00000000000000..a15646f4dad127 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-06-21-11-53-09.gh-issue-65210.PhFRBJ.rst @@ -0,0 +1,3 @@ +Change the declaration of the *keywords* parameter of +:c:func:`PyArg_ParseTupleAndKeywords` and +:c:func:`PyArg_VaParseTupleAndKeywords` for better compatibility with C++. diff --git a/Misc/NEWS.d/next/C API/2023-11-15-01-26-59.gh-issue-111545.iAoFtA.rst b/Misc/NEWS.d/next/C API/2023-11-15-01-26-59.gh-issue-111545.iAoFtA.rst new file mode 100644 index 00000000000000..7bde2498acf999 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-15-01-26-59.gh-issue-111545.iAoFtA.rst @@ -0,0 +1,2 @@ +Add :c:func:`Py_HashPointer` function to hash a pointer. Patch by Victor +Stinner. diff --git a/Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst b/Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst new file mode 100644 index 00000000000000..113119efd6aebb --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-11-27-09-44-16.gh-issue-112438.GdNZiI.rst @@ -0,0 +1,2 @@ +Fix support of format units "es", "et", "es#", and "et#" in nested tuples in +:c:func:`PyArg_ParseTuple`-like functions. diff --git a/Misc/NEWS.d/next/C API/2023-12-02-02-08-11.gh-issue-106560.THvuji.rst b/Misc/NEWS.d/next/C API/2023-12-02-02-08-11.gh-issue-106560.THvuji.rst new file mode 100644 index 00000000000000..59b461ec47ad64 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-12-02-02-08-11.gh-issue-106560.THvuji.rst @@ -0,0 +1,2 @@ +Fix redundant declarations in the public C API. Declare PyBool_Type, +PyLong_Type and PySys_Audit() only once. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-13-13-25-15.bpo-34392.9kIlMF.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-13-13-25-15.bpo-34392.9kIlMF.rst new file mode 100644 index 00000000000000..bc4fd1ad1f5c7c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-13-13-25-15.bpo-34392.9kIlMF.rst @@ -0,0 +1 @@ +Added :func:`sys._is_interned`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst new file mode 100644 index 00000000000000..5201ab7d842088 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-07-07-05-37-53.gh-issue-94606.hojJ54.rst @@ -0,0 +1,3 @@ +Fix UnicodeEncodeError when :func:`email.message.get_payload` reads a message +with a Unicode surrogate character and the message content is not well-formed for +surrogateescape encoding. Patch by Sidney Markowitz. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst new file mode 100644 index 00000000000000..0da2fd33b0ea52 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-22-13-17-54.gh-issue-112320.EddM51.rst @@ -0,0 +1,4 @@ +The Tier 2 translator now tracks the confidence level for staying "on trace" +(i.e. not exiting back to the Tier 1 interpreter) for branch instructions +based on the number of bits set in the branch "counter". Trace translation +ends when the confidence drops below 1/3rd. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-24-14-10-57.gh-issue-112367.9z1IDp.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-24-14-10-57.gh-issue-112367.9z1IDp.rst new file mode 100644 index 00000000000000..991e45ad47fabe --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-24-14-10-57.gh-issue-112367.9z1IDp.rst @@ -0,0 +1,2 @@ +Avoid undefined behaviour when using the perf trampolines by not freeing the +code arenas until shutdown. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst new file mode 100644 index 00000000000000..adac11bf4c90a1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-39-44.gh-issue-112387.AbBq5W.rst @@ -0,0 +1,2 @@ +Fix error positions for decoded strings with backwards tokenize errors. +Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst new file mode 100644 index 00000000000000..1c82be2febda4f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-25-22-58-49.gh-issue-112388.MU3cIM.rst @@ -0,0 +1,2 @@ +Fix an error that was causing the parser to try to overwrite tokenizer +errors. Patch by pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-26-21-30-11.gh-issue-111058.q4DqDY.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-26-21-30-11.gh-issue-111058.q4DqDY.rst new file mode 100644 index 00000000000000..de5661f911aa82 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-26-21-30-11.gh-issue-111058.q4DqDY.rst @@ -0,0 +1,3 @@ +Change coro.cr_frame/gen.gi_frame to return ``None`` after the coroutine/generator has been closed. +This fixes a bug where :func:`~inspect.getcoroutinestate` and :func:`~inspect.getgeneratorstate` +return the wrong state for a closed coroutine/generator. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst new file mode 100644 index 00000000000000..d4efbab6b2d128 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-27-18-55-30.gh-issue-112217.SwFLMj.rst @@ -0,0 +1 @@ +Add check for the type of ``__cause__`` returned from calling the type ``T`` in ``raise from T``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-01-08-16-10.gh-issue-95754.ae4gwy.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-01-08-16-10.gh-issue-95754.ae4gwy.rst new file mode 100644 index 00000000000000..0884bc4a4be726 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-01-08-16-10.gh-issue-95754.ae4gwy.rst @@ -0,0 +1 @@ +Provide a better error message when accessing invalid attributes on partially initialized modules. The origin of the module being accessed is now included in the message to help with the common issue of shadowing other modules. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-01-19-02-21.gh-issue-105967.Puq5Cn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-01-19-02-21.gh-issue-105967.Puq5Cn.rst new file mode 100644 index 00000000000000..c69511218e3e16 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-01-19-02-21.gh-issue-105967.Puq5Cn.rst @@ -0,0 +1,4 @@ +Workaround a bug in Apple's macOS platform zlib library where +:func:`zlib.crc32` and :func:`binascii.crc32` could produce incorrect results +on multi-gigabyte inputs. Including when using :mod:`zipfile` on zips +containing large data. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-03-15-29-53.gh-issue-112660.gldBvh.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-03-15-29-53.gh-issue-112660.gldBvh.rst new file mode 100644 index 00000000000000..ea9052b3e35c48 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-03-15-29-53.gh-issue-112660.gldBvh.rst @@ -0,0 +1,2 @@ +Do not clear unexpected errors during formatting error messages for +ImportError and AttributeError for modules. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-03-19-34-51.gh-issue-112625.QWTlwS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-03-19-34-51.gh-issue-112625.QWTlwS.rst new file mode 100644 index 00000000000000..4970e10f3f4dcb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-03-19-34-51.gh-issue-112625.QWTlwS.rst @@ -0,0 +1 @@ +Fixes a bug where a bytearray object could be cleared while iterating over an argument in the ``bytearray.join()`` method that could result in reading memory after it was freed. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-04-23-09-07.gh-issue-112730.BXHlFa.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-04-23-09-07.gh-issue-112730.BXHlFa.rst new file mode 100644 index 00000000000000..51758dd5f4c318 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-04-23-09-07.gh-issue-112730.BXHlFa.rst @@ -0,0 +1 @@ +Use color to highlight error locations in tracebacks. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst new file mode 100644 index 00000000000000..44d63269c5424a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-05-20-41-58.gh-issue-112716.hOcx0Y.rst @@ -0,0 +1,2 @@ +Fix SystemError in the ``import`` statement and in ``__reduce__()`` methods +of builtin types when ``__builtins__`` is not a dict. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-07-12-00-04.gh-issue-74616.kgTGVb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-07-12-00-04.gh-issue-74616.kgTGVb.rst new file mode 100644 index 00000000000000..5c345be9de6d0b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-07-12-00-04.gh-issue-74616.kgTGVb.rst @@ -0,0 +1,2 @@ +:func:`input` now raises a ValueError when output on the terminal if the +prompt contains embedded null characters instead of silently truncating it. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-07-13-19-55.gh-issue-112125.4ADN7i.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-07-13-19-55.gh-issue-112125.4ADN7i.rst new file mode 100644 index 00000000000000..52cd45029fb8c7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-07-13-19-55.gh-issue-112125.4ADN7i.rst @@ -0,0 +1 @@ +Fix None.__ne__(None) returning NotImplemented instead of False diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst new file mode 100644 index 00000000000000..4bc2fe7c26d904 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-00-50-00.gh-issue-112943.RHNZie.rst @@ -0,0 +1,2 @@ +Correctly compute end column offsets for multiline tokens in the +:mod:`tokenize` module. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst new file mode 100644 index 00000000000000..6b7881bbd19f59 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-11-19-53-32.gh-issue-90350.-FQy3E.rst @@ -0,0 +1 @@ +Optimize builtin functions :func:`min` and :func:`max`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst new file mode 100644 index 00000000000000..96606924d4a3ec --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-12-04-53-19.gh-issue-108866.xbJ-9a.rst @@ -0,0 +1,3 @@ +Change the API and contract of ``_PyExecutorObject`` to return the +next_instr pointer, instead of the frame, and to always execute at least one +instruction. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-13-11-45-53.gh-issue-106905.5dslTN.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-13-11-45-53.gh-issue-106905.5dslTN.rst new file mode 100644 index 00000000000000..e3a772f3354ecf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-13-11-45-53.gh-issue-106905.5dslTN.rst @@ -0,0 +1,7 @@ +Use per AST-parser state rather than global state to track recursion depth +within the AST parser to prevent potential race condition due to +simultaneous parsing. + +The issue primarily showed up in 3.11 by multithreaded users of +:func:`ast.parse`. In 3.12 a change to when garbage collection can be +triggered prevented the race condition from occurring. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-14-20-08-35.gh-issue-113054.e20CtM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-14-20-08-35.gh-issue-113054.e20CtM.rst new file mode 100644 index 00000000000000..d0729f9c44754c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-14-20-08-35.gh-issue-113054.e20CtM.rst @@ -0,0 +1,2 @@ +Fixed bug where a redundant NOP is not removed, causing an assertion to fail +in the compiler in debug mode. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst new file mode 100644 index 00000000000000..01ca1cc7f79b8f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst @@ -0,0 +1,3 @@ +Increase the C recursion limit by a factor of 3 for non-debug builds, except +for webassembly and s390 platforms which are unchanged. This mitigates some +regressions in 3.12 with deep recursion mixing builtin (C) and Python code. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-19-22-03-43.gh-issue-111375.M9vuA6.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-19-22-03-43.gh-issue-111375.M9vuA6.rst new file mode 100644 index 00000000000000..fbb517173451f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-19-22-03-43.gh-issue-111375.M9vuA6.rst @@ -0,0 +1,2 @@ +Only use ``NULL`` in the exception stack to indicate an exception was +handled. Patch by Carey Metcalfe. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-20-08-54-54.gh-issue-113212.62AUlw.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-20-08-54-54.gh-issue-113212.62AUlw.rst new file mode 100644 index 00000000000000..6edbc9c60d968c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-20-08-54-54.gh-issue-113212.62AUlw.rst @@ -0,0 +1 @@ +Improve :py:class:`super` error messages. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-20-18-27-11.gh-issue-113297.BZyAI_.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-20-18-27-11.gh-issue-113297.BZyAI_.rst new file mode 100644 index 00000000000000..b6aee1f241fd23 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-20-18-27-11.gh-issue-113297.BZyAI_.rst @@ -0,0 +1 @@ +Fix segfault in the compiler on with statement with 19 context managers. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-01-00-07-02.gh-issue-113602.cWuTzk.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-01-00-07-02.gh-issue-113602.cWuTzk.rst new file mode 100644 index 00000000000000..5e064657348720 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-01-00-07-02.gh-issue-113602.cWuTzk.rst @@ -0,0 +1,2 @@ +Fix an error that was causing the parser to try to overwrite existing errors +and crashing in the process. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-01-23-57-24.gh-issue-113603.ySwovr.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-01-23-57-24.gh-issue-113603.ySwovr.rst new file mode 100644 index 00000000000000..5fe6d80dedd19d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-01-23-57-24.gh-issue-113603.ySwovr.rst @@ -0,0 +1 @@ +Fixed bug where a redundant NOP is not removed, causing an assertion to fail in the compiler in debug mode. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-02-11-14-29.gh-issue-113657.CQo9vF.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-02-11-14-29.gh-issue-113657.CQo9vF.rst new file mode 100644 index 00000000000000..b520b5c2529425 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-02-11-14-29.gh-issue-113657.CQo9vF.rst @@ -0,0 +1,2 @@ +Fix an issue that caused important instruction pointer updates to be +optimized out of tier two traces. diff --git a/Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst b/Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst new file mode 100644 index 00000000000000..215db7beb75dcf --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-10-23-23-43-43.gh-issue-110746.yg77IE.rst @@ -0,0 +1 @@ +Improved markup for valid options/values for methods ttk.treeview.column and ttk.treeview.heading, and for Layouts. diff --git a/Misc/NEWS.d/next/Documentation/2023-11-30-02-33-59.gh-issue-111699._O5G_y.rst b/Misc/NEWS.d/next/Documentation/2023-11-30-02-33-59.gh-issue-111699._O5G_y.rst new file mode 100644 index 00000000000000..2d31345e6c2044 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-11-30-02-33-59.gh-issue-111699._O5G_y.rst @@ -0,0 +1 @@ +Relocate ``smtpd`` deprecation notice to its own section rather than under ``locale`` in What's New in Python 3.12 document diff --git a/Misc/NEWS.d/next/IDLE/2019-12-13-12-26-56.bpo-13586.1grqsR.rst b/Misc/NEWS.d/next/IDLE/2019-12-13-12-26-56.bpo-13586.1grqsR.rst new file mode 100644 index 00000000000000..1a73cad175c888 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-12-13-12-26-56.bpo-13586.1grqsR.rst @@ -0,0 +1 @@ +Enter the selected text when opening the "Replace" dialog. diff --git a/Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst b/Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst new file mode 100644 index 00000000000000..1c20e46b1e5f7b --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-12-10-20-01-11.gh-issue-112898.98aWv2.rst @@ -0,0 +1 @@ +Fix processing unsaved files when quitting IDLE on macOS. diff --git a/Misc/NEWS.d/next/IDLE/2023-12-19-00-03-12.gh-issue-113269.lrU-IC.rst b/Misc/NEWS.d/next/IDLE/2023-12-19-00-03-12.gh-issue-113269.lrU-IC.rst new file mode 100644 index 00000000000000..72e75b7910e359 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-12-19-00-03-12.gh-issue-113269.lrU-IC.rst @@ -0,0 +1 @@ +Fix test_editor hang on macOS Catalina. diff --git a/Misc/NEWS.d/next/Library/2019-02-12-16-12-54.bpo-21360.gkSSfx.rst b/Misc/NEWS.d/next/Library/2019-02-12-16-12-54.bpo-21360.gkSSfx.rst new file mode 100644 index 00000000000000..bc32b9fe4199f9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-02-12-16-12-54.bpo-21360.gkSSfx.rst @@ -0,0 +1 @@ +:class:`mailbox.Maildir` now ignores files with a leading dot. diff --git a/Misc/NEWS.d/next/Library/2019-05-17-07-22-33.bpo-18060.5mqTQM.rst b/Misc/NEWS.d/next/Library/2019-05-17-07-22-33.bpo-18060.5mqTQM.rst new file mode 100644 index 00000000000000..3fefbc3efb63c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-17-07-22-33.bpo-18060.5mqTQM.rst @@ -0,0 +1,2 @@ +Fixed a class inheritance issue that can cause segfaults when deriving two or more levels of subclasses from a base class of Structure or Union. + diff --git a/Misc/NEWS.d/next/Library/2019-05-18-15-50-14.bpo-36959.ew6WZ4.rst b/Misc/NEWS.d/next/Library/2019-05-18-15-50-14.bpo-36959.ew6WZ4.rst new file mode 100644 index 00000000000000..1ac05a730a2086 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-18-15-50-14.bpo-36959.ew6WZ4.rst @@ -0,0 +1,2 @@ +Fix some error messages for invalid ISO format string combinations in ``strptime()`` that referred to directives not contained in the format string. +Patch by Gordon P. Hemsley. diff --git a/Misc/NEWS.d/next/Library/2019-06-14-22-37-32.bpo-37260.oecdIf.rst b/Misc/NEWS.d/next/Library/2019-06-14-22-37-32.bpo-37260.oecdIf.rst new file mode 100644 index 00000000000000..a5f2c5e8e18919 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-06-14-22-37-32.bpo-37260.oecdIf.rst @@ -0,0 +1,2 @@ +Fixed a race condition in :func:`shutil.rmtree` in which directory entries removed by another process or thread while ``shutil.rmtree()`` is running can cause it to raise FileNotFoundError. Patch by Jeffrey Kintscher. + diff --git a/Misc/NEWS.d/next/Library/2020-03-09-15-08-29.bpo-39912.xPOBBY.rst b/Misc/NEWS.d/next/Library/2020-03-09-15-08-29.bpo-39912.xPOBBY.rst new file mode 100644 index 00000000000000..fb8579725a2d7d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-03-09-15-08-29.bpo-39912.xPOBBY.rst @@ -0,0 +1,3 @@ +:func:`warnings.filterwarnings()` and :func:`warnings.simplefilter()` now raise +appropriate exceptions instead of ``AssertionError``. Patch contributed by +Rémi Lapeyre. diff --git a/Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst b/Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst new file mode 100644 index 00000000000000..c017a1c8df09d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-21-23-32-46.bpo-40262.z4fQv1.rst @@ -0,0 +1,2 @@ +The :meth:`ssl.SSLSocket.recv_into` method no longer requires the *buffer* +argument to implement ``__len__`` and supports buffers with arbitrary item size. diff --git a/Misc/NEWS.d/next/Library/2020-06-15-23-44-53.bpo-19821.ihBk39.rst b/Misc/NEWS.d/next/Library/2020-06-15-23-44-53.bpo-19821.ihBk39.rst new file mode 100644 index 00000000000000..ede68106b56ff8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-06-15-23-44-53.bpo-19821.ihBk39.rst @@ -0,0 +1 @@ +The :func:`!pydoc.ispackage` function has been deprecated. diff --git a/Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst b/Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst new file mode 100644 index 00000000000000..8bde68f8f2afc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-07-28-20-48-05.bpo-41422.iMwnMu.rst @@ -0,0 +1,2 @@ +Fixed memory leaks of :class:`pickle.Pickler` and :class:`pickle.Unpickler` involving cyclic references via the +internal memo mapping. diff --git a/Misc/NEWS.d/next/Library/2020-08-06-14-43-55.bpo-26791.KxoEfO.rst b/Misc/NEWS.d/next/Library/2020-08-06-14-43-55.bpo-26791.KxoEfO.rst new file mode 100644 index 00000000000000..c6f8dcb6f9269c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-08-06-14-43-55.bpo-26791.KxoEfO.rst @@ -0,0 +1,4 @@ +:func:`shutil.move` now moves a symlink into a directory when that +directory is the target of the symlink. This provides the same behavior as +the mv shell command. The previous behavior raised an exception. Patch by +Jeffrey Kintscher. diff --git a/Misc/NEWS.d/next/Library/2020-12-14-09-31-13.bpo-35332.s22wAx.rst b/Misc/NEWS.d/next/Library/2020-12-14-09-31-13.bpo-35332.s22wAx.rst new file mode 100644 index 00000000000000..80564b99a079c6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-12-14-09-31-13.bpo-35332.s22wAx.rst @@ -0,0 +1,3 @@ +The :func:`shutil.rmtree` function now ignores errors when calling +:func:`os.close` when *ignore_errors* is ``True``, and +:func:`os.close` no longer retried after error. diff --git a/Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst b/Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst new file mode 100644 index 00000000000000..92f3b870c11131 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-11-23-22-22-49.bpo-32731.kNOASr.rst @@ -0,0 +1,3 @@ +:func:`getpass.getuser` now raises :exc:`OSError` for all failures rather +than :exc:`ImportError` on systems lacking the :mod:`pwd` module or +:exc:`KeyError` if the password database is empty. diff --git a/Misc/NEWS.d/next/Library/2021-12-06-22-10-53.bpo-43153.J7mjSy.rst b/Misc/NEWS.d/next/Library/2021-12-06-22-10-53.bpo-43153.J7mjSy.rst new file mode 100644 index 00000000000000..7800e0a4869adf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-12-06-22-10-53.bpo-43153.J7mjSy.rst @@ -0,0 +1,4 @@ +On Windows, ``tempfile.TemporaryDirectory`` previously masked a +``PermissionError`` with ``NotADirectoryError`` during directory cleanup. It +now correctly raises ``PermissionError`` if errors are not ignored. Patch by +Andrei Kulakov and Ken Jin. diff --git a/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst b/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst new file mode 100644 index 00000000000000..7991048fc48e03 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst @@ -0,0 +1,2 @@ +Fix a bug in :class:`tempfile.TemporaryDirectory` cleanup, which now no longer +dereferences symlinks when working around file system permission errors. diff --git a/Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst b/Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst new file mode 100644 index 00000000000000..6074dd7f101a6d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-08-00-43-29.gh-issue-83162.ufdI9F.rst @@ -0,0 +1,3 @@ +Renamed :exc:`!re.error` to :exc:`PatternError` for clarity, and kept +:exc:`!re.error` for backward compatibility. Patch by Matthias Bussonnier and +Adam Chhina. diff --git a/Misc/NEWS.d/next/Library/2023-04-09-21-05-43.gh-issue-66515.0DS8Ya.rst b/Misc/NEWS.d/next/Library/2023-04-09-21-05-43.gh-issue-66515.0DS8Ya.rst new file mode 100644 index 00000000000000..b9c52f3b8db52c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-09-21-05-43.gh-issue-66515.0DS8Ya.rst @@ -0,0 +1,3 @@ +:class:`mailbox.MH` now supports folders that do not contain a +``.mh_sequences`` file (e.g. Claws Mail IMAP-cache folders). Patch by Serhiy +Storchaka. diff --git a/Misc/NEWS.d/next/Library/2023-04-23-11-08-02.gh-issue-103708.Y17C7p.rst b/Misc/NEWS.d/next/Library/2023-04-23-11-08-02.gh-issue-103708.Y17C7p.rst new file mode 100644 index 00000000000000..4b7d747175df03 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-23-11-08-02.gh-issue-103708.Y17C7p.rst @@ -0,0 +1 @@ +Make hardcoded python name, a configurable parameter so that different implementations of python can override it instead of making huge diffs in sysconfig.py diff --git a/Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst b/Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst new file mode 100644 index 00000000000000..82d61ca8b8bc97 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst @@ -0,0 +1,3 @@ +Add :func:`warnings.deprecated`, a decorator to mark deprecated functions to +static type checkers and to warn on usage of deprecated classes and functions. +See :pep:`702`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2023-08-07-21-11-24.gh-issue-102130._UyI5i.rst b/Misc/NEWS.d/next/Library/2023-08-07-21-11-24.gh-issue-102130._UyI5i.rst new file mode 100644 index 00000000000000..f582ad5df39e84 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-07-21-11-24.gh-issue-102130._UyI5i.rst @@ -0,0 +1 @@ +Support tab completion in :mod:`cmd` for ``editline``. diff --git a/Misc/NEWS.d/next/Library/2023-08-14-21-10-52.gh-issue-103363.u64_QI.rst b/Misc/NEWS.d/next/Library/2023-08-14-21-10-52.gh-issue-103363.u64_QI.rst new file mode 100644 index 00000000000000..d4a27d624eb5e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-21-10-52.gh-issue-103363.u64_QI.rst @@ -0,0 +1,2 @@ +Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.owner` +and :meth:`~pathlib.Path.group`, defaulting to ``True``. diff --git a/Misc/NEWS.d/next/Library/2023-09-23-14-40-51.gh-issue-109786.UX3pKv.rst b/Misc/NEWS.d/next/Library/2023-09-23-14-40-51.gh-issue-109786.UX3pKv.rst new file mode 100644 index 00000000000000..07222fa339d703 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-23-14-40-51.gh-issue-109786.UX3pKv.rst @@ -0,0 +1,2 @@ +Fix possible reference leaks and crash when re-enter the ``__next__()`` method of +:class:`itertools.pairwise`. diff --git a/Misc/NEWS.d/next/Library/2023-10-11-02-34-01.gh-issue-110109.RFCmHs.rst b/Misc/NEWS.d/next/Library/2023-10-11-02-34-01.gh-issue-110109.RFCmHs.rst new file mode 100644 index 00000000000000..4f12d128f49fb3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-11-02-34-01.gh-issue-110109.RFCmHs.rst @@ -0,0 +1,3 @@ +Add private ``pathlib._PurePathBase`` class: a base class for +:class:`pathlib.PurePath` that omits certain magic methods. It may be made +public (along with ``_PathBase``) in future. diff --git a/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst b/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst new file mode 100644 index 00000000000000..d7e6b225489b99 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-12-18-19-47.gh-issue-82300.P8-O38.rst @@ -0,0 +1 @@ +Add ``track`` parameter to :class:`multiprocessing.shared_memory.SharedMemory` that allows using shared memory blocks without having to register with the POSIX resource tracker that automatically releases them upon process exit. diff --git a/Misc/NEWS.d/next/Library/2023-10-17-16-11-03.gh-issue-52161.WBYyCJ.rst b/Misc/NEWS.d/next/Library/2023-10-17-16-11-03.gh-issue-52161.WBYyCJ.rst new file mode 100644 index 00000000000000..3f598d40e4ae93 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-17-16-11-03.gh-issue-52161.WBYyCJ.rst @@ -0,0 +1,2 @@ +:meth:`cmd.Cmd.do_help` now cleans docstrings with :func:`inspect.cleandoc` +before writing them. Patch by Filip Łapkiewicz. diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst new file mode 100644 index 00000000000000..3d0e9e4078c934 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst @@ -0,0 +1,8 @@ +:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now +return ``('', '')`` 2-tuples in more situations where invalid email +addresses are encountered instead of potentially inaccurate values. Add +optional *strict* parameter to these two functions: use ``strict=False`` to +get the old behavior, accept malformed inputs. +``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check +if the *strict* paramater is available. Patch by Thomas Dwyer and Victor +Stinner to improve the CVE-2023-27043 fix. diff --git a/Misc/NEWS.d/next/Library/2023-10-23-03-49-34.gh-issue-102980.aXBd54.rst b/Misc/NEWS.d/next/Library/2023-10-23-03-49-34.gh-issue-102980.aXBd54.rst new file mode 100644 index 00000000000000..d4bae4790d6fa4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-23-03-49-34.gh-issue-102980.aXBd54.rst @@ -0,0 +1 @@ +Redirect the output of ``interact`` command of :mod:`pdb` to the same channel as the debugger. Add tests and improve docs. diff --git a/Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst b/Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst new file mode 100644 index 00000000000000..b1de348bea0a58 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-23-18-42-26.gh-issue-111049.Ys7-o_.rst @@ -0,0 +1,2 @@ +Fix crash during garbage collection of the :class:`io.BytesIO` buffer +object. diff --git a/Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst b/Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst new file mode 100644 index 00000000000000..44c5702a6551b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-25-13-07-53.gh-issue-67790.jMn9Ad.rst @@ -0,0 +1,2 @@ +Implement basic formatting support (minimum width, alignment, fill) for +:class:`fractions.Fraction`. diff --git a/Misc/NEWS.d/next/Library/2023-10-25-16-37-13.gh-issue-75666.BpsWut.rst b/Misc/NEWS.d/next/Library/2023-10-25-16-37-13.gh-issue-75666.BpsWut.rst new file mode 100644 index 00000000000000..d774cc4f7c687f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-25-16-37-13.gh-issue-75666.BpsWut.rst @@ -0,0 +1,6 @@ +Fix the behavior of :mod:`tkinter` widget's ``unbind()`` method with two +arguments. Previously, ``widget.unbind(sequence, funcid)`` destroyed the +current binding for *sequence*, leaving *sequence* unbound, and deleted the +*funcid* command. Now it removes only *funcid* from the binding for +*sequence*, keeping other commands, and deletes the *funcid* command. It +leaves *sequence* unbound only if *funcid* was the last bound command. diff --git a/Misc/NEWS.d/next/Library/2023-11-02-10-13-31.gh-issue-111615.3SMixi.rst b/Misc/NEWS.d/next/Library/2023-11-02-10-13-31.gh-issue-111615.3SMixi.rst new file mode 100644 index 00000000000000..f80ab00a3adbff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-02-10-13-31.gh-issue-111615.3SMixi.rst @@ -0,0 +1,2 @@ +Fix a regression caused by a fix to gh-93162 whereby you couldn't configure +a :class:`QueueHandler` without specifying handlers. diff --git a/Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst b/Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst new file mode 100644 index 00000000000000..0920da221e423f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-05-20-09-27.gh-issue-99367.HLaWKo.rst @@ -0,0 +1 @@ +Do not mangle ``sys.path[0]`` in :mod:`pdb` if safe_path is set diff --git a/Misc/NEWS.d/next/Library/2023-11-08-16-11-04.gh-issue-110275.Bm6GwR.rst b/Misc/NEWS.d/next/Library/2023-11-08-16-11-04.gh-issue-110275.Bm6GwR.rst new file mode 100644 index 00000000000000..194dd5cb623f0f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-08-16-11-04.gh-issue-110275.Bm6GwR.rst @@ -0,0 +1,2 @@ +Named tuple's methods ``_replace()`` and ``__replace__()`` now raise +TypeError instead of ValueError for invalid keyword arguments. diff --git a/Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst b/Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst new file mode 100644 index 00000000000000..30379b8fa1afaf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-08-18-53-07.gh-issue-68166.1iTh4Y.rst @@ -0,0 +1,2 @@ +Add support of the "vsapi" element type in +:meth:`tkinter.ttk.Style.element_create`. diff --git a/Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst b/Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst new file mode 100644 index 00000000000000..50408202a7a5a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-09-11-07-34.gh-issue-111874.dzYc3j.rst @@ -0,0 +1,4 @@ +When creating a :class:`typing.NamedTuple` class, ensure +:func:`~object.__set_name__` is called on all objects that define +``__set_name__`` and exist in the values of the ``NamedTuple`` class's class +dictionary. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-11-15-01-36-04.gh-issue-106922.qslOVH.rst b/Misc/NEWS.d/next/Library/2023-11-15-01-36-04.gh-issue-106922.qslOVH.rst new file mode 100644 index 00000000000000..b68e75ab87cd0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-15-01-36-04.gh-issue-106922.qslOVH.rst @@ -0,0 +1 @@ +Display multiple lines with ``traceback`` when errors span multiple lines. diff --git a/Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst b/Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst new file mode 100644 index 00000000000000..4243dcb190434f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-15-04-53-37.gh-issue-112105.I3RcVN.rst @@ -0,0 +1 @@ +Make :func:`readline.set_completer_delims` work with libedit diff --git a/Misc/NEWS.d/next/Library/2023-11-16-10-42-15.gh-issue-112139.WpHosf.rst b/Misc/NEWS.d/next/Library/2023-11-16-10-42-15.gh-issue-112139.WpHosf.rst new file mode 100644 index 00000000000000..090dc8847d9556 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-16-10-42-15.gh-issue-112139.WpHosf.rst @@ -0,0 +1,3 @@ +Add :meth:`Signature.format` to format signatures to string with extra options. +And use it in :mod:`pydoc` to render more readable signatures that have new +lines between parameters. diff --git a/Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst b/Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst new file mode 100644 index 00000000000000..8345e33791cde0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-22-19-43-54.gh-issue-112292.5nDU87.rst @@ -0,0 +1,2 @@ +Fix a crash in :mod:`readline` when imported from a sub interpreter. Patch +by Anthony Shaw diff --git a/Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst b/Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst new file mode 100644 index 00000000000000..bd686ad052e5b2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-23-10-41-21.gh-issue-112332.rhTBaa.rst @@ -0,0 +1,2 @@ +Deprecate the ``exc_type`` field of :class:`traceback.TracebackException`. +Add ``exc_type_str`` to replace it. diff --git a/Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst b/Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst new file mode 100644 index 00000000000000..c222febae6b554 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-28-02-39-30.gh-issue-101336.ya433z.rst @@ -0,0 +1 @@ +Add ``keep_alive`` keyword parameter for :meth:`AbstractEventLoop.create_server` and :meth:`BaseEventLoop.create_server`. diff --git a/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst b/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst new file mode 100644 index 00000000000000..a16d67e7776bcb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst @@ -0,0 +1,3 @@ +Fix edge cases that could cause a key to be present in both the +``__required_keys__`` and ``__optional_keys__`` attributes of a +:class:`typing.TypedDict`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2023-11-28-20-47-39.gh-issue-112328.Z2AxEY.rst b/Misc/NEWS.d/next/Library/2023-11-28-20-47-39.gh-issue-112328.Z2AxEY.rst new file mode 100644 index 00000000000000..6e6902486b7bc9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-28-20-47-39.gh-issue-112328.Z2AxEY.rst @@ -0,0 +1,2 @@ +[Enum] Make ``EnumDict``, ``EnumDict.member_names``, +``EnumType._add_alias_`` and ``EnumType._add_value_alias_`` public. diff --git a/Misc/NEWS.d/next/Library/2023-11-29-02-26-32.gh-issue-112510.j-zXGc.rst b/Misc/NEWS.d/next/Library/2023-11-29-02-26-32.gh-issue-112510.j-zXGc.rst new file mode 100644 index 00000000000000..02de6fa80c1b3e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-29-02-26-32.gh-issue-112510.j-zXGc.rst @@ -0,0 +1 @@ +Add :data:`readline.backend` for the backend readline uses (``editline`` or ``readline``) diff --git a/Misc/NEWS.d/next/Library/2023-11-29-10-51-41.gh-issue-112516.rFKUKN.rst b/Misc/NEWS.d/next/Library/2023-11-29-10-51-41.gh-issue-112516.rFKUKN.rst new file mode 100644 index 00000000000000..530cf992dcd77a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-29-10-51-41.gh-issue-112516.rFKUKN.rst @@ -0,0 +1 @@ +Update the bundled copy of pip to version 23.3.1. diff --git a/Misc/NEWS.d/next/Library/2023-12-01-08-28-09.gh-issue-112578.bfNbfi.rst b/Misc/NEWS.d/next/Library/2023-12-01-08-28-09.gh-issue-112578.bfNbfi.rst new file mode 100644 index 00000000000000..1de5b1fe26ce6d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-01-08-28-09.gh-issue-112578.bfNbfi.rst @@ -0,0 +1 @@ +Fix a spurious :exc:`RuntimeWarning` when executing the :mod:`zipfile` module. diff --git a/Misc/NEWS.d/next/Library/2023-12-01-16-09-59.gh-issue-81194.FFad1c.rst b/Misc/NEWS.d/next/Library/2023-12-01-16-09-59.gh-issue-81194.FFad1c.rst new file mode 100644 index 00000000000000..feb7a8643b97f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-01-16-09-59.gh-issue-81194.FFad1c.rst @@ -0,0 +1,3 @@ +Fix a crash in :func:`socket.if_indextoname` with specific value (UINT_MAX). +Fix an integer overflow in :func:`socket.if_indextoname` on 64-bit +non-Windows platforms. diff --git a/Misc/NEWS.d/next/Library/2023-12-01-18-05-09.gh-issue-110190.5bf-c9.rst b/Misc/NEWS.d/next/Library/2023-12-01-18-05-09.gh-issue-110190.5bf-c9.rst new file mode 100644 index 00000000000000..730b9d49119805 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-01-18-05-09.gh-issue-110190.5bf-c9.rst @@ -0,0 +1 @@ +Fix ctypes structs with array on Arm platform by setting ``MAX_STRUCT_SIZE`` to 32 in stgdict. Patch by Diego Russo. diff --git a/Misc/NEWS.d/next/Library/2023-12-01-21-05-46.gh-issue-112334.DmNXKh.rst b/Misc/NEWS.d/next/Library/2023-12-01-21-05-46.gh-issue-112334.DmNXKh.rst new file mode 100644 index 00000000000000..3a53a8bf84230f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-01-21-05-46.gh-issue-112334.DmNXKh.rst @@ -0,0 +1,11 @@ +Fixed a performance regression in 3.12's :mod:`subprocess` on Linux where it +would no longer use the fast-path ``vfork()`` system call when it could have +due to a logic bug, instead falling back to the safe but slower ``fork()``. + +Also fixed a second 3.12.0 potential security bug. If a value of +``extra_groups=[]`` was passed to :mod:`subprocess.Popen` or related APIs, +the underlying ``setgroups(0, NULL)`` system call to clear the groups list +would not be made in the child process prior to ``exec()``. + +This was identified via code inspection in the process of fixing the first +bug. diff --git a/Misc/NEWS.d/next/Library/2023-12-02-12-55-17.gh-issue-112618.7_FT8-.rst b/Misc/NEWS.d/next/Library/2023-12-02-12-55-17.gh-issue-112618.7_FT8-.rst new file mode 100644 index 00000000000000..c732de15609c96 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-02-12-55-17.gh-issue-112618.7_FT8-.rst @@ -0,0 +1,2 @@ +Fix a caching bug relating to :data:`typing.Annotated`. +``Annotated[str, True]`` is no longer identical to ``Annotated[str, 1]``. diff --git a/Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst b/Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst new file mode 100644 index 00000000000000..91c88bac334dcb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-03-01-01-52.gh-issue-112622.1Z8cpx.rst @@ -0,0 +1,2 @@ +Ensure ``name`` parameter is passed to event loop in +:func:`asyncio.create_task`. diff --git a/Misc/NEWS.d/next/Library/2023-12-03-12-41-48.gh-issue-112645.blMsKf.rst b/Misc/NEWS.d/next/Library/2023-12-03-12-41-48.gh-issue-112645.blMsKf.rst new file mode 100644 index 00000000000000..4e8f6ebdb882e0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-03-12-41-48.gh-issue-112645.blMsKf.rst @@ -0,0 +1 @@ +Remove deprecation error on passing ``onerror`` to :func:`shutil.rmtree`. diff --git a/Misc/NEWS.d/next/Library/2023-12-04-14-05-24.gh-issue-74690.eODKRm.rst b/Misc/NEWS.d/next/Library/2023-12-04-14-05-24.gh-issue-74690.eODKRm.rst new file mode 100644 index 00000000000000..36d793f787302e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-04-14-05-24.gh-issue-74690.eODKRm.rst @@ -0,0 +1,5 @@ +Speedup :func:`isinstance` checks by roughly 20% for +:func:`runtime-checkable protocols ` +that only have one callable member. +Speedup :func:`issubclass` checks for these protocols by roughly 10%. +Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-12-04-16-45-11.gh-issue-74690.pQYP5U.rst b/Misc/NEWS.d/next/Library/2023-12-04-16-45-11.gh-issue-74690.pQYP5U.rst new file mode 100644 index 00000000000000..8102f02e941c29 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-04-16-45-11.gh-issue-74690.pQYP5U.rst @@ -0,0 +1,2 @@ +Speedup :func:`issubclass` checks against simple :func:`runtime-checkable +protocols ` by around 6%. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-12-04-21-30-34.gh-issue-112727.jpgNRB.rst b/Misc/NEWS.d/next/Library/2023-12-04-21-30-34.gh-issue-112727.jpgNRB.rst new file mode 100644 index 00000000000000..bbe7aae5732d9a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-04-21-30-34.gh-issue-112727.jpgNRB.rst @@ -0,0 +1 @@ +Speed up :meth:`pathlib.Path.absolute`. Patch by Barney Gale. diff --git a/Misc/NEWS.d/next/Library/2023-12-05-01-19-28.gh-issue-112736.rdHDrU.rst b/Misc/NEWS.d/next/Library/2023-12-05-01-19-28.gh-issue-112736.rdHDrU.rst new file mode 100644 index 00000000000000..6c09e622923af8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-05-01-19-28.gh-issue-112736.rdHDrU.rst @@ -0,0 +1 @@ +The use of del-safe symbols in ``subprocess`` was refactored to allow for use in cross-platform build environments. diff --git a/Misc/NEWS.d/next/Library/2023-12-05-16-20-40.gh-issue-94692.-e5C3c.rst b/Misc/NEWS.d/next/Library/2023-12-05-16-20-40.gh-issue-94692.-e5C3c.rst new file mode 100644 index 00000000000000..c67ba6c9ececdb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-05-16-20-40.gh-issue-94692.-e5C3c.rst @@ -0,0 +1,4 @@ +:func:`shutil.rmtree` now only catches OSError exceptions. Previously a +symlink attack resistant version of ``shutil.rmtree()`` could ignore or pass +to the error handler arbitrary exception when invalid arguments were +provided. diff --git a/Misc/NEWS.d/next/Library/2023-12-05-18-57-53.gh-issue-79325.P2vMVK.rst b/Misc/NEWS.d/next/Library/2023-12-05-18-57-53.gh-issue-79325.P2vMVK.rst new file mode 100644 index 00000000000000..f3c32d27b5fe66 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-05-18-57-53.gh-issue-79325.P2vMVK.rst @@ -0,0 +1,2 @@ +Fix an infinite recursion error in :func:`tempfile.TemporaryDirectory` +cleanup on Windows. diff --git a/Misc/NEWS.d/next/Library/2023-12-06-14-06-14.gh-issue-51944.-5qq_L.rst b/Misc/NEWS.d/next/Library/2023-12-06-14-06-14.gh-issue-51944.-5qq_L.rst new file mode 100644 index 00000000000000..821eefa7cffcd5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-06-14-06-14.gh-issue-51944.-5qq_L.rst @@ -0,0 +1,6 @@ +Add the following constants to the :mod:`termios` module. These values are +present in macOS system headers: ``ALTWERASE``, ``B14400``, ``B28800``, +``B7200``, ``B76800``, ``CCAR_OFLOW``, ``CCTS_OFLOW``, ``CDSR_OFLOW``, +``CDTR_IFLOW``, ``CIGNORE``, ``CRTS_IFLOW``, ``EXTPROC``, ``IUTF8``, +``MDMBUF``, ``NL2``, ``NL3``, ``NOKERNINFO``, ``ONOEOT``, ``OXTABS``, +``VDSUSP``, ``VSTATUS``. diff --git a/Misc/NEWS.d/next/Library/2023-12-06-16-01-33.gh-issue-112800.TNsGJ-.rst b/Misc/NEWS.d/next/Library/2023-12-06-16-01-33.gh-issue-112800.TNsGJ-.rst new file mode 100644 index 00000000000000..e88eac169177a9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-06-16-01-33.gh-issue-112800.TNsGJ-.rst @@ -0,0 +1,2 @@ +Fix :mod:`asyncio` ``SubprocessTransport.close()`` not to throw +``PermissionError`` when used with setuid executables. diff --git a/Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst b/Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst new file mode 100644 index 00000000000000..bfeec3c95207cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-07-16-55-41.gh-issue-87286.MILC9_.rst @@ -0,0 +1,3 @@ +Added :const:`LOG_FTP`, :const:`LOG_NETINFO`, :const:`LOG_REMOTEAUTH`, +:const:`LOG_INSTALL`, :const:`LOG_RAS`, and :const:`LOG_LAUNCHD` tot the +:mod:`syslog` module, all of them constants on used on macOS. diff --git a/Misc/NEWS.d/next/Library/2023-12-08-11-17-17.gh-issue-112540.Pm5egX.rst b/Misc/NEWS.d/next/Library/2023-12-08-11-17-17.gh-issue-112540.Pm5egX.rst new file mode 100644 index 00000000000000..263b13d1762bf1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-08-11-17-17.gh-issue-112540.Pm5egX.rst @@ -0,0 +1,2 @@ +The statistics.geometric_mean() function now returns zero for datasets +containing a zero. Formerly, it would raise an exception. diff --git a/Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst b/Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst new file mode 100644 index 00000000000000..3bfed1e0f1dc91 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-11-14-12-46.gh-issue-110190.e0iEUa.rst @@ -0,0 +1 @@ +Fix ctypes structs with array on PPC64LE platform by setting ``MAX_STRUCT_SIZE`` to 64 in stgdict. Patch by Diego Russo. diff --git a/Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst b/Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst new file mode 100644 index 00000000000000..58ca26af511383 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-11-16-13-15.gh-issue-112970.87jmKP.rst @@ -0,0 +1 @@ +Use :c:func:`!closefrom` on Linux where available (e.g. glibc-2.34), rather than only FreeBSD. diff --git a/Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst b/Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst new file mode 100644 index 00000000000000..ceeab8cc7d6bec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-12-05-48-17.gh-issue-112989.ZAa_eq.rst @@ -0,0 +1 @@ +Reduce overhead to connect sockets with :mod:`asyncio` SelectorEventLoop. diff --git a/Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst b/Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst new file mode 100644 index 00000000000000..b99e6bc90ae791 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-12-16-32-55.gh-issue-112962.ZZWXZn.rst @@ -0,0 +1,3 @@ +:mod:`dis` module functions add cache information to the +:class:`~dis.Instruction` instance rather than creating fake +:class:`~dis.Instruction` instances to represent the cache entries. diff --git a/Misc/NEWS.d/next/Library/2023-12-12-20-15-57.gh-issue-112559.IgXkje.rst b/Misc/NEWS.d/next/Library/2023-12-12-20-15-57.gh-issue-112559.IgXkje.rst new file mode 100644 index 00000000000000..c08cb7c3ba5ea5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-12-20-15-57.gh-issue-112559.IgXkje.rst @@ -0,0 +1,3 @@ +:func:`signal.signal` and :func:`signal.getsignal` no longer call ``repr`` on +callable handlers. :func:`asyncio.run` and :meth:`asyncio.Runner.run` no longer +call ``repr`` on the task results. Patch by Yilei Yang. diff --git a/Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst b/Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst new file mode 100644 index 00000000000000..793ae63b4c1ff5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-13-17-08-21.gh-issue-59616.JNlWSs.rst @@ -0,0 +1,3 @@ +Add support of :func:`os.lchmod` and the *follow_symlinks* argument in +:func:`os.chmod` on Windows. Note that the default value of *follow_symlinks* +in :func:`!os.lchmod` is ``False`` on Windows. diff --git a/Misc/NEWS.d/next/Library/2023-12-15-09-51-41.gh-issue-113175.RHsNwE.rst b/Misc/NEWS.d/next/Library/2023-12-15-09-51-41.gh-issue-113175.RHsNwE.rst new file mode 100644 index 00000000000000..1b43803d1a7aa4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-15-09-51-41.gh-issue-113175.RHsNwE.rst @@ -0,0 +1,5 @@ +Sync with importlib_metadata 7.0, including improved type annotations, fixed +issue with symlinked packages in ``package_distributions``, added +``EntryPoints.__repr__``, introduced the ``diagnose`` script, added +``Distribution.origin`` property, and removed deprecated ``EntryPoint`` +access by numeric index (tuple behavior). diff --git a/Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst b/Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst new file mode 100644 index 00000000000000..c841e5c7f7683a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-15-12-35-28.gh-issue-61648.G-4pz0.rst @@ -0,0 +1 @@ +Detect line numbers of properties in doctests. diff --git a/Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst b/Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst new file mode 100644 index 00000000000000..44f26aef60a33a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-15-18-10-26.gh-issue-113202.xv_Ww8.rst @@ -0,0 +1 @@ +Add a ``strict`` option to ``batched()`` in the ``itertools`` module. diff --git a/Misc/NEWS.d/next/Library/2023-12-15-18-13-59.gh-issue-113119.al-569.rst b/Misc/NEWS.d/next/Library/2023-12-15-18-13-59.gh-issue-113119.al-569.rst new file mode 100644 index 00000000000000..94087b00515e97 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-15-18-13-59.gh-issue-113119.al-569.rst @@ -0,0 +1,2 @@ +:func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned +process use the current process environment. Patch by Jakub Kulik. diff --git a/Misc/NEWS.d/next/Library/2023-12-15-20-29-49.gh-issue-113188.AvoraB.rst b/Misc/NEWS.d/next/Library/2023-12-15-20-29-49.gh-issue-113188.AvoraB.rst new file mode 100644 index 00000000000000..17c69572d9f2b1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-15-20-29-49.gh-issue-113188.AvoraB.rst @@ -0,0 +1,6 @@ +Fix :func:`shutil.copymode` and :func:`shutil.copystat` on Windows. +Previously they worked differenly if *dst* is a symbolic link: +they modified the permission bits of *dst* itself +rather than the file it points to if *follow_symlinks* is true or *src* is +not a symbolic link, and did not modify the permission bits if +*follow_symlinks* is false and *src* is a symbolic link. diff --git a/Misc/NEWS.d/next/Library/2023-12-15-21-33-42.gh-issue-113191.Il155b.rst b/Misc/NEWS.d/next/Library/2023-12-15-21-33-42.gh-issue-113191.Il155b.rst new file mode 100644 index 00000000000000..13fe4ff5f6a8bd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-15-21-33-42.gh-issue-113191.Il155b.rst @@ -0,0 +1,2 @@ +Add support of :func:`os.fchmod` and a file descriptor in :func:`os.chmod` +on Windows. diff --git a/Misc/NEWS.d/next/Library/2023-12-16-01-10-47.gh-issue-113199.oDjnjL.rst b/Misc/NEWS.d/next/Library/2023-12-16-01-10-47.gh-issue-113199.oDjnjL.rst new file mode 100644 index 00000000000000..d8e0b1731d1e3b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-16-01-10-47.gh-issue-113199.oDjnjL.rst @@ -0,0 +1,3 @@ +Make ``http.client.HTTPResponse.read1`` and +``http.client.HTTPResponse.readline`` close IO after reading all data when +content length is known. Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Library/2023-12-16-10-58-34.gh-issue-113117.0zF7bH.rst b/Misc/NEWS.d/next/Library/2023-12-16-10-58-34.gh-issue-113117.0zF7bH.rst new file mode 100644 index 00000000000000..718226a0021efe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-16-10-58-34.gh-issue-113117.0zF7bH.rst @@ -0,0 +1,4 @@ +The :mod:`subprocess` module can now use the :func:`os.posix_spawn` function +with ``close_fds=True`` on platforms where +``posix_spawn_file_actions_addclosefrom_np`` is available. +Patch by Jakub Kulik. diff --git a/Misc/NEWS.d/next/Library/2023-12-16-23-56-42.gh-issue-113149.7LWgTS.rst b/Misc/NEWS.d/next/Library/2023-12-16-23-56-42.gh-issue-113149.7LWgTS.rst new file mode 100644 index 00000000000000..0faa67fefabeca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-16-23-56-42.gh-issue-113149.7LWgTS.rst @@ -0,0 +1,2 @@ +Improve error message when a JSON array or object contains a trailing comma. +Patch by Carson Radtke. diff --git a/Misc/NEWS.d/next/Library/2023-12-17-13-56-30.gh-issue-87264.RgfHCv.rst b/Misc/NEWS.d/next/Library/2023-12-17-13-56-30.gh-issue-87264.RgfHCv.rst new file mode 100644 index 00000000000000..fa987d4f0af9ba --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-17-13-56-30.gh-issue-87264.RgfHCv.rst @@ -0,0 +1 @@ +Fixed tarfile list() method to show file type. diff --git a/Misc/NEWS.d/next/Library/2023-12-18-09-47-54.gh-issue-113246.em930H.rst b/Misc/NEWS.d/next/Library/2023-12-18-09-47-54.gh-issue-113246.em930H.rst new file mode 100644 index 00000000000000..167bb37c0e0643 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-18-09-47-54.gh-issue-113246.em930H.rst @@ -0,0 +1 @@ +Update bundled pip to 23.3.2. diff --git a/Misc/NEWS.d/next/Library/2023-12-20-21-18-51.gh-issue-113214.JcV9Mn.rst b/Misc/NEWS.d/next/Library/2023-12-20-21-18-51.gh-issue-113214.JcV9Mn.rst new file mode 100644 index 00000000000000..6db74cda166e92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-20-21-18-51.gh-issue-113214.JcV9Mn.rst @@ -0,0 +1 @@ +Fix an ``AttributeError`` during asyncio SSL protocol aborts in SSL-over-SSL scenarios. diff --git a/Misc/NEWS.d/next/Library/2023-12-21-23-47-42.gh-issue-53502.dercJI.rst b/Misc/NEWS.d/next/Library/2023-12-21-23-47-42.gh-issue-53502.dercJI.rst new file mode 100644 index 00000000000000..aa7274161d4166 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-21-23-47-42.gh-issue-53502.dercJI.rst @@ -0,0 +1,2 @@ +Add a new option ``aware_datetime`` in :mod:`plistlib` to loads or dumps +aware datetime. diff --git a/Misc/NEWS.d/next/Library/2023-12-22-20-49-52.gh-issue-113407.C_O13_.rst b/Misc/NEWS.d/next/Library/2023-12-22-20-49-52.gh-issue-113407.C_O13_.rst new file mode 100644 index 00000000000000..da00977f03cefd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-22-20-49-52.gh-issue-113407.C_O13_.rst @@ -0,0 +1 @@ +Fix import of :mod:`unittest.mock` when CPython is built without docstrings. diff --git a/Misc/NEWS.d/next/Library/2023-12-23-13-10-42.gh-issue-111784.Nb4L1j.rst b/Misc/NEWS.d/next/Library/2023-12-23-13-10-42.gh-issue-111784.Nb4L1j.rst new file mode 100644 index 00000000000000..51ac0752cfae84 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-23-13-10-42.gh-issue-111784.Nb4L1j.rst @@ -0,0 +1,5 @@ +Fix segfaults in the ``_elementtree`` module. +Fix first segfault during deallocation of ``_elementtree.XMLParser`` instances by keeping strong reference +to ``pyexpat`` module in module state for capsule lifetime. +Fix second segfault which happens in the same deallocation process by keeping strong reference +to ``_elementtree`` module in ``XMLParser`` structure for ``_elementtree`` module lifetime. diff --git a/Misc/NEWS.d/next/Library/2023-12-23-16-10-07.gh-issue-113421.w7vs08.rst b/Misc/NEWS.d/next/Library/2023-12-23-16-10-07.gh-issue-113421.w7vs08.rst new file mode 100644 index 00000000000000..2082fe6391d261 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-23-16-10-07.gh-issue-113421.w7vs08.rst @@ -0,0 +1 @@ +Fix multiprocessing logger for ``%(filename)s``. diff --git a/Misc/NEWS.d/next/Library/2023-12-23-16-51-17.gh-issue-113028.3Jmdoj.rst b/Misc/NEWS.d/next/Library/2023-12-23-16-51-17.gh-issue-113028.3Jmdoj.rst new file mode 100644 index 00000000000000..5f66d6a00b4d3d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-23-16-51-17.gh-issue-113028.3Jmdoj.rst @@ -0,0 +1,6 @@ +When a second reference to a string appears in the input to :mod:`pickle`, +and the Python implementation is in use, +we are guaranteed that a single copy gets pickled +and a single object is shared when reloaded. +Previously, in protocol 0, when a string contained certain characters +(e.g. newline) it resulted in duplicate objects. diff --git a/Misc/NEWS.d/next/Library/2023-12-28-14-36-20.gh-issue-113543.2iWkOR.rst b/Misc/NEWS.d/next/Library/2023-12-28-14-36-20.gh-issue-113543.2iWkOR.rst new file mode 100644 index 00000000000000..5bf557bedd0204 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-28-14-36-20.gh-issue-113543.2iWkOR.rst @@ -0,0 +1,2 @@ +Make sure that ``webbrowser.MacOSXOSAScript`` sends ``webbrowser.open`` +audit event. diff --git a/Misc/NEWS.d/next/Security/2023-12-06-14-06-59.gh-issue-112302.3bl20f.rst b/Misc/NEWS.d/next/Security/2023-12-06-14-06-59.gh-issue-112302.3bl20f.rst new file mode 100644 index 00000000000000..65e4dc3762d3c0 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-12-06-14-06-59.gh-issue-112302.3bl20f.rst @@ -0,0 +1,2 @@ +Created a Software Bill-of-Materials document and tooling for tracking +dependencies. diff --git a/Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst b/Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst new file mode 100644 index 00000000000000..8fbe42d263feb9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2020-05-16-18-00-21.bpo-40648.p2uPqy.rst @@ -0,0 +1 @@ +Test modes that file can get with chmod() on Windows. diff --git a/Misc/NEWS.d/next/Tests/2023-09-05-20-46-35.gh-issue-108927.TpwWav.rst b/Misc/NEWS.d/next/Tests/2023-09-05-20-46-35.gh-issue-108927.TpwWav.rst new file mode 100644 index 00000000000000..b1a78370afedb2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-05-20-46-35.gh-issue-108927.TpwWav.rst @@ -0,0 +1,4 @@ +Fixed order dependence in running tests in the same process +when a test that has submodules (e.g. test_importlib) follows a test that +imports its submodule (e.g. test_importlib.util) and precedes a test +(e.g. test_unittest or test_compileall) that uses that submodule. diff --git a/Misc/NEWS.d/next/Tests/2023-12-04-15-56-11.gh-issue-112334.FFc9Ti.rst b/Misc/NEWS.d/next/Tests/2023-12-04-15-56-11.gh-issue-112334.FFc9Ti.rst new file mode 100644 index 00000000000000..aeaad6e5055522 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-12-04-15-56-11.gh-issue-112334.FFc9Ti.rst @@ -0,0 +1,2 @@ +Adds a regression test to verify that ``vfork()`` is used when expected by +:mod:`subprocess` on vfork enabled POSIX systems (Linux). diff --git a/Misc/NEWS.d/next/Tests/2023-12-05-19-50-03.gh-issue-112769.kdLJmS.rst b/Misc/NEWS.d/next/Tests/2023-12-05-19-50-03.gh-issue-112769.kdLJmS.rst new file mode 100644 index 00000000000000..1bbbb26fc322fa --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-12-05-19-50-03.gh-issue-112769.kdLJmS.rst @@ -0,0 +1,3 @@ +The tests now correctly compare zlib version when +:const:`zlib.ZLIB_RUNTIME_VERSION` contains non-integer suffixes. For +example zlib-ng defines the version as ``1.3.0.zlib-ng``. diff --git a/Misc/NEWS.d/next/Tests/2023-12-09-21-27-46.gh-issue-109980.y--500.rst b/Misc/NEWS.d/next/Tests/2023-12-09-21-27-46.gh-issue-109980.y--500.rst new file mode 100644 index 00000000000000..c475a33919db98 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-12-09-21-27-46.gh-issue-109980.y--500.rst @@ -0,0 +1,2 @@ +Fix ``test_tarfile_vs_tar`` in ``test_shutil`` for macOS, where system tar +can include more information in the archive than :mod:`shutil.make_archive`. diff --git a/Misc/NEWS.d/next/Tests/2024-01-01-14-40-02.gh-issue-113633.VOY5ai.rst b/Misc/NEWS.d/next/Tests/2024-01-01-14-40-02.gh-issue-113633.VOY5ai.rst new file mode 100644 index 00000000000000..150c0d91852cdf --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-01-01-14-40-02.gh-issue-113633.VOY5ai.rst @@ -0,0 +1 @@ +Use module state for the _testcapi extension module. diff --git a/Misc/NEWS.d/next/Windows/2023-08-08-01-42-14.gh-issue-73427.WOpiNt.rst b/Misc/NEWS.d/next/Windows/2023-08-08-01-42-14.gh-issue-73427.WOpiNt.rst new file mode 100644 index 00000000000000..830c4c54838e80 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-08-08-01-42-14.gh-issue-73427.WOpiNt.rst @@ -0,0 +1,2 @@ +Deprecate :func:`sys._enablelegacywindowsfsencoding`. Use +:envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. Patch by Inada Naoki. diff --git a/Misc/NEWS.d/next/Windows/2023-12-03-19-22-37.gh-issue-112278.FiloCE.rst b/Misc/NEWS.d/next/Windows/2023-12-03-19-22-37.gh-issue-112278.FiloCE.rst new file mode 100644 index 00000000000000..0350d105d97375 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-03-19-22-37.gh-issue-112278.FiloCE.rst @@ -0,0 +1,2 @@ +Reduce the time cost for some functions in :mod:`platform` on Windows if +current user has no permission to the WMI. diff --git a/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst b/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst new file mode 100644 index 00000000000000..5a3493356e30be --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-05-22-56-30.gh-issue-111650.xlWmvM.rst @@ -0,0 +1,3 @@ +Ensures the ``Py_GIL_DISABLED`` preprocessor variable is defined in +:file:`pyconfig.h` so that extension modules written in C are able to use +it. diff --git a/Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst b/Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst new file mode 100644 index 00000000000000..cf2883357a962a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-11-20-23-04.gh-issue-71383.9pZh6t.rst @@ -0,0 +1,2 @@ +Update Tcl/Tk in Windows installer to 8.6.13 with a patch to suppress +incorrect ThemeChanged warnings. diff --git a/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst b/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst new file mode 100644 index 00000000000000..c1d96792bdae0b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst @@ -0,0 +1 @@ +Fixes path calculations when launching Python on Windows through a symlink. diff --git a/Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst b/Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst new file mode 100644 index 00000000000000..6fd7f7f9afdfa2 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-14-19-00-29.gh-issue-113009.6LNdjz.rst @@ -0,0 +1,5 @@ +:mod:`multiprocessing`: On Windows, fix a race condition in +``Process.terminate()``: no longer set the ``returncode`` attribute to +always call ``WaitForSingleObject()`` in ``Process.wait()``. Previously, +sometimes the process was still running after ``TerminateProcess()`` even if +``GetExitCodeProcess()`` is not ``STILL_ACTIVE``. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Windows/2023-12-19-10-56-46.gh-issue-111973.A9Wtsb.rst b/Misc/NEWS.d/next/Windows/2023-12-19-10-56-46.gh-issue-111973.A9Wtsb.rst new file mode 100644 index 00000000000000..0cefa4e44093f0 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-19-10-56-46.gh-issue-111973.A9Wtsb.rst @@ -0,0 +1 @@ +Update Windows installer to use SQLite 3.44.2. diff --git a/Misc/NEWS.d/next/macOS/2023-12-06-12-11-13.gh-issue-109981.mOHg10.rst b/Misc/NEWS.d/next/macOS/2023-12-06-12-11-13.gh-issue-109981.mOHg10.rst new file mode 100644 index 00000000000000..f86ab2c37ee6ec --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-06-12-11-13.gh-issue-109981.mOHg10.rst @@ -0,0 +1,3 @@ +Use ``/dev/fd`` on macOS to determine the number of open files in +``test.support.os_helper.fd_count`` to avoid a crash with "guarded" file +descriptors when probing for open files. diff --git a/Misc/NEWS.d/next/macOS/2023-12-07-14-19-46.gh-issue-110820.DIxb_F.rst b/Misc/NEWS.d/next/macOS/2023-12-07-14-19-46.gh-issue-110820.DIxb_F.rst new file mode 100644 index 00000000000000..0badace7928745 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-07-14-19-46.gh-issue-110820.DIxb_F.rst @@ -0,0 +1,3 @@ +Make sure the preprocessor definitions for ``ALIGNOF_MAX_ALIGN_T``, +``SIZEOF_LONG_DOUBLE`` and ``HAVE_GCC_ASM_FOR_X64`` are correct for +Universal 2 builds on macOS. diff --git a/Misc/NEWS.d/next/macOS/2023-12-07-15-53-16.gh-issue-110017.UMYzMR.rst b/Misc/NEWS.d/next/macOS/2023-12-07-15-53-16.gh-issue-110017.UMYzMR.rst new file mode 100644 index 00000000000000..eab1746f1ae3f7 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-07-15-53-16.gh-issue-110017.UMYzMR.rst @@ -0,0 +1,2 @@ +Disable a signal handling stress test on macOS due to a bug in macOS +(FB13453490). diff --git a/Misc/NEWS.d/next/macOS/2023-12-10-20-30-06.gh-issue-102362.y8svbF.rst b/Misc/NEWS.d/next/macOS/2023-12-10-20-30-06.gh-issue-102362.y8svbF.rst new file mode 100644 index 00000000000000..55c5ac01434660 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-10-20-30-06.gh-issue-102362.y8svbF.rst @@ -0,0 +1,3 @@ +Make sure the result of :func:`sysconfig.get_plaform` includes at least a +major and minor versions, even if ``MACOSX_DEPLOYMENT_TARGET`` is set to +only a major version during build to match the format expected by pip. diff --git a/Misc/NEWS.d/next/macOS/2023-12-16-11-45-32.gh-issue-108269.wVgCHF.rst b/Misc/NEWS.d/next/macOS/2023-12-16-11-45-32.gh-issue-108269.wVgCHF.rst new file mode 100644 index 00000000000000..85598454abcaad --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-16-11-45-32.gh-issue-108269.wVgCHF.rst @@ -0,0 +1,4 @@ +Set ``CFBundleAllowMixedLocalizations`` to true in the Info.plist for the +framework, embedded Python.app and IDLE.app with framework installs on +macOS. This allows applications to pick up the user's preferred locale when +that's different from english. diff --git a/Misc/NEWS.d/next/macOS/2023-12-19-10-50-08.gh-issue-111973.HMHJfy.rst b/Misc/NEWS.d/next/macOS/2023-12-19-10-50-08.gh-issue-111973.HMHJfy.rst new file mode 100644 index 00000000000000..0cf3abf3b71890 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-19-10-50-08.gh-issue-111973.HMHJfy.rst @@ -0,0 +1 @@ +Update macOS installer to use SQLite 3.44.2. diff --git a/Misc/NEWS.d/next/macOS/2023-12-21-09-41-42.gh-issue-87277.IF6EZZ.rst b/Misc/NEWS.d/next/macOS/2023-12-21-09-41-42.gh-issue-87277.IF6EZZ.rst new file mode 100644 index 00000000000000..4ae55c0293198a --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-21-09-41-42.gh-issue-87277.IF6EZZ.rst @@ -0,0 +1,3 @@ +webbrowser: Don't look for X11 browsers on macOS. Those are generally not +used and probing for them can result in starting XQuartz even if it isn't +used otherwise. diff --git a/Misc/NEWS.d/next/macOS/2023-12-21-10-20-41.gh-issue-65701.Q2hNbN.rst b/Misc/NEWS.d/next/macOS/2023-12-21-10-20-41.gh-issue-65701.Q2hNbN.rst new file mode 100644 index 00000000000000..870b84a4d1af80 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-21-10-20-41.gh-issue-65701.Q2hNbN.rst @@ -0,0 +1,2 @@ +The :program:`freeze` tool doesn't work with framework builds of Python. +Document this and bail out early when running the tool with such a build. diff --git a/Misc/NEWS.d/next/macOS/2023-12-21-11-53-47.gh-issue-74573.MA6Vys.rst b/Misc/NEWS.d/next/macOS/2023-12-21-11-53-47.gh-issue-74573.MA6Vys.rst new file mode 100644 index 00000000000000..96dcd4765d95da --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-21-11-53-47.gh-issue-74573.MA6Vys.rst @@ -0,0 +1,3 @@ +Document that :mod:`dbm.ndbm` can silently corrupt DBM files on updates when +exceeding undocumented platform limits, and can crash (segmentation fault) +when reading such a corrupted file. (FB8919203) diff --git a/Misc/NEWS.d/next/macOS/2023-12-23-22-41-07.gh-issue-110459.NaMBJy.rst b/Misc/NEWS.d/next/macOS/2023-12-23-22-41-07.gh-issue-110459.NaMBJy.rst new file mode 100644 index 00000000000000..44ffd857785f0d --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-23-22-41-07.gh-issue-110459.NaMBJy.rst @@ -0,0 +1,2 @@ +Running ``configure ... --with-openssl-rpath=X/Y/Z`` no longer fails to detect +OpenSSL on macOS. diff --git a/Misc/NEWS.d/next/macOS/2023-12-28-12-18-39.gh-issue-113536.0ythg7.rst b/Misc/NEWS.d/next/macOS/2023-12-28-12-18-39.gh-issue-113536.0ythg7.rst new file mode 100644 index 00000000000000..828b872d283627 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-28-12-18-39.gh-issue-113536.0ythg7.rst @@ -0,0 +1 @@ +:func:`os.waitid` is now available on macOS diff --git a/Misc/python.man b/Misc/python.man index 9f89c94adf5028..14cbd85c60bfa9 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -601,6 +601,9 @@ show how long each import takes. This is exactly equivalent to setting .IP PYTHONBREAKPOINT If this environment variable is set to 0, it disables the default debugger. It can be set to the callable of your debugger of choice. +.IP PYTHON_COLORS +If this variable is set to 1, the interpreter will colorize various kinds of +output. Setting it to 0 deactivates this behavior. .SS Debug-mode variables Setting these variables only has an effect in a debug build of Python, that is, if Python was configured with the diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json new file mode 100644 index 00000000000000..5b3cd04ffa7f74 --- /dev/null +++ b/Misc/sbom.spdx.json @@ -0,0 +1,2294 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "files": [ + { + "SPDXID": "SPDXRef-FILE-Modules-expat-COPYING", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "39e6f567a10e36b2e77727e98e60bbcb3eb3af0b" + }, + { + "algorithm": "SHA256", + "checksumValue": "122f2c27000472a201d337b9b31f7eb2b52d091b02857061a8880371612d9534" + } + ], + "fileName": "Modules/expat/COPYING" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-ascii.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b0235fa3cf845a7d68e8e66dd344d5e32e8951b5" + }, + { + "algorithm": "SHA256", + "checksumValue": "42f8b392c70366743eacbc60ce021389ccaa333598dd49eef6ee5c93698ca205" + } + ], + "fileName": "Modules/expat/ascii.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-asciitab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cbb53d16ca1f35ee9c9e296116efd222ae611ed9" + }, + { + "algorithm": "SHA256", + "checksumValue": "1cc0ae749019fc0e488cd1cf245f6beaa6d4f7c55a1fc797e5aa40a408bc266b" + } + ], + "fileName": "Modules/expat/asciitab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-expat.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ab7bb32514d170592dfb3f76e41bbdc075a4e7e0" + }, + { + "algorithm": "SHA256", + "checksumValue": "f521acdad222644365b0e81a33bcd6939a98c91b225c47582cc84bd73d96febc" + } + ], + "fileName": "Modules/expat/expat.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-expat-config.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "73627287302ee3e84347c4fe21f37a9cb828bc3b" + }, + { + "algorithm": "SHA256", + "checksumValue": "f17e59f9d95eeb05694c02508aa284d332616c22cbe2e6a802d8a0710310eaab" + } + ], + "fileName": "Modules/expat/expat_config.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-expat-external.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b70ce53fdc25ae482681ae2f6623c3c8edc9c1b7" + }, + { + "algorithm": "SHA256", + "checksumValue": "86afb425ec9999eb4f1ec9ab2fb41c58c4aa5cb9bf934b8c94264670fc5a961d" + } + ], + "fileName": "Modules/expat/expat_external.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-iasciitab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "1b0e9014c0baa4c6254d2b5e6a67c70148309c34" + }, + { + "algorithm": "SHA256", + "checksumValue": "ad8b01e9f323cc4208bcd22241df383d7e8641fe3c8b3415aa513de82531f89f" + } + ], + "fileName": "Modules/expat/iasciitab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-internal.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2790d37e7de2f13dccc4f4fb352cbdf9ed6abaa2" + }, + { + "algorithm": "SHA256", + "checksumValue": "d2efe5a1018449968a689f444cca432e3d5875aba6ad08ee18ca235d64f41bb9" + } + ], + "fileName": "Modules/expat/internal.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-latin1tab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d335ecca380e331a0ea7dc33838a4decd93ec1e4" + }, + { + "algorithm": "SHA256", + "checksumValue": "eab66226da100372e01e42e1cbcd8ac2bbbb5c1b5f95d735289cc85c7a8fc2ba" + } + ], + "fileName": "Modules/expat/latin1tab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-nametab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cf2bc9626c945826602ba9170786e9a2a44645e4" + }, + { + "algorithm": "SHA256", + "checksumValue": "67dcf415d37a4b692a6a8bb46f990c02d83f2ef3d01a65cd61c8594a084246f2" + } + ], + "fileName": "Modules/expat/nametab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-pyexpatns.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "baa44fe4581895d42e8d5e83d8ce6a69b1c34dbe" + }, + { + "algorithm": "SHA256", + "checksumValue": "33a7b9ac8bf4571e23272cdf644c6f9808bd44c66b149e3c41ab3870d1888609" + } + ], + "fileName": "Modules/expat/pyexpatns.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-siphash.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2b984f806f10fbfbf72d8d1b7ba2992413c15299" + }, + { + "algorithm": "SHA256", + "checksumValue": "fbce56cd680e690043bbf572188cc2d0a25dbfc0d47ac8cb98eb3de768d4e694" + } + ], + "fileName": "Modules/expat/siphash.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-utf8tab.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b77c8fcfb551553c81d6fbd94c798c8aa04ad021" + }, + { + "algorithm": "SHA256", + "checksumValue": "8cd26bd461d334d5e1caedb3af4518d401749f2fc66d56208542b29085159c18" + } + ], + "fileName": "Modules/expat/utf8tab.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-winconfig.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "e774ae6ee9391aa6ffb8f775fb74e48f4b428959" + }, + { + "algorithm": "SHA256", + "checksumValue": "3c71cea9a6174718542331971a35db317902b2433be9d8dd1cb24239b635c0cc" + } + ], + "fileName": "Modules/expat/winconfig.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmlparse.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b580e827e16baa6b035586ffcd4d90301e5a353f" + }, + { + "algorithm": "SHA256", + "checksumValue": "483518bbd69338eefc706cd7fc0b6039df2d3e347f64097989059ed6d2385a1e" + } + ], + "fileName": "Modules/expat/xmlparse.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmlrole.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "5ef21312af73deb2428be3fe97a65244608e76de" + }, + { + "algorithm": "SHA256", + "checksumValue": "6fcf8c72ac0112c1b98bd2039c632a66b4c3dc516ce7c1f981390951121ef3c0" + } + ], + "fileName": "Modules/expat/xmlrole.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmlrole.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "c1a4ea6356643d0820edb9c024c20ad2aaf562dc" + }, + { + "algorithm": "SHA256", + "checksumValue": "2b5d674be6ef20c7e3f69295176d75e68c5616e4dfce0a186fdd5e2ed8315f7a" + } + ], + "fileName": "Modules/expat/xmlrole.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "e6d66ae9fd61d7950c62c5d87693c30a707e8577" + }, + { + "algorithm": "SHA256", + "checksumValue": "1110f651bdccfa765ad3d6f3857a35887ab35fc0fe7f3f3488fde2b238b482e3" + } + ], + "fileName": "Modules/expat/xmltok.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "9c2a544875fd08ba9c2397296c97263518a410aa" + }, + { + "algorithm": "SHA256", + "checksumValue": "4299a03828b98bfe47ec6809f6e279252954a9a911dc7e0f19551bd74e3af971" + } + ], + "fileName": "Modules/expat/xmltok.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok-impl.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "aa96882de8e3d1d3083124b595aa911efe44e5ad" + }, + { + "algorithm": "SHA256", + "checksumValue": "0fbcba7931707c60301305dab78d2298d96447d0a5513926d8b18135228c0818" + } + ], + "fileName": "Modules/expat/xmltok_impl.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok-impl.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "788332fe8040bed71172cddedb69abd848cc62f7" + }, + { + "algorithm": "SHA256", + "checksumValue": "f05ad4fe5e98429a7349ff04f57192cac58c324601f2a2e5e697ab0bc05d36d5" + } + ], + "fileName": "Modules/expat/xmltok_impl.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xmltok-ns.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2d82d0a1201f78d478b30d108ff8fc27ee3e2672" + }, + { + "algorithm": "SHA256", + "checksumValue": "6ce6d03193279078d55280150fe91e7370370b504a6c123a79182f28341f3e90" + } + ], + "fileName": "Modules/expat/xmltok_ns.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f77449b2b4eb99f1da0938633cc558baf9c444fb" + }, + { + "algorithm": "SHA256", + "checksumValue": "0f252967debca5b35362ca53951ea16ca8bb97a19a1d24f6695f44d50010859e" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_MD5.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "c24e6779a91c840f3d65d24abbce225b608b676e" + }, + { + "algorithm": "SHA256", + "checksumValue": "9cd062e782801013e3cacaba583e44e1b5e682e217d20208d5323354d42011f1" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_MD5.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "560f6ff541b5eff480ea047b147f4212bb0db7ed" + }, + { + "algorithm": "SHA256", + "checksumValue": "0ade3ab264e912d7b4e5cdcf773db8c63e4440540d295922d74b06bcfc74c77a" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA1.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "853b77d45379146faaeac5fe899b28db386ad13c" + }, + { + "algorithm": "SHA256", + "checksumValue": "b13eb14f91582703819235ea7c8f807bb93e4f1e6b695499dc1d86021dc39e72" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA1.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "667120b6100c946cdaa442f1173c723339923071" + }, + { + "algorithm": "SHA256", + "checksumValue": "b189459b863341a3a9c5c78c0208b6554a2f2ac26e0748fbd4432a91db21fae6" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "81db38b0b920e63ec33c7109d1144c35cf091da0" + }, + { + "algorithm": "SHA256", + "checksumValue": "631c9ba19c1c2c835bb63d3f2f22b8d76fb535edfed3c254ff2a52f12af3fe61" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "9c832b98a2f2a68202d2da016fb718965d7b7602" + }, + { + "algorithm": "SHA256", + "checksumValue": "38d350d1184238966cfa821a59ae00343f362182b6c2fbea7f2651763d757fb7" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ecc766fb6f7ee85e902b593b61b41e5a728fca34" + }, + { + "algorithm": "SHA256", + "checksumValue": "bae290a94366a2460f51e8468144baaade91d9048db111e10d2e2ffddc3f98cf" + } + ], + "fileName": "Modules/_hacl/Hacl_Hash_SHA3.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ab7b4d9465a2765a07f8d5bccace7182b28ed1b8" + }, + { + "algorithm": "SHA256", + "checksumValue": "26913613f3b4f8ffff0a3e211a5ebc849159094e5e11de0a31fcb95b6105b74c" + } + ], + "fileName": "Modules/_hacl/Hacl_Streaming_Types.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2ea61d6a236147462045f65c20311819d74db80c" + }, + { + "algorithm": "SHA256", + "checksumValue": "2c22b4d49ba06d6a3053cdc66405bd5ae953a28fcfed1ab164e8f5e0f6e2fb8b" + } + ], + "fileName": "Modules/_hacl/include/krml/FStar_UInt128_Verified.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "1a647d841180ac8ca667afa968c353425e81ad0d" + }, + { + "algorithm": "SHA256", + "checksumValue": "e5d1c5854833bec7ea02e227ec35bd7b49c5fb9e0f339efa0dd83e1595f722d4" + } + ], + "fileName": "Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "1987119a563a8fdc5966286e274f716dbcea77ee" + }, + { + "algorithm": "SHA256", + "checksumValue": "fe57e1bc5ce3224d106e36cb8829b5399c63a68a70b0ccd0c91d82a4565c8869" + } + ], + "fileName": "Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "903c9eb76b01f3a95c04c3bc841c2fb71dea5403" + }, + { + "algorithm": "SHA256", + "checksumValue": "08ec602c7f90a1540389c0cfc95769fa7fec251e7ca143ef83c0b9f7afcf89a7" + } + ], + "fileName": "Modules/_hacl/include/krml/internal/target.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "964e09bd99ff2366afd6193b59863fc925e7fb05" + }, + { + "algorithm": "SHA256", + "checksumValue": "3734c7942bec9a434e16df069fa45bdcb84b130f14417bc5f7bfe8546272d9f5" + } + ], + "fileName": "Modules/_hacl/include/krml/lowstar_endianness.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "df8e0ed74a5970d09d3cc4c6e7c6c7a4c4e5015c" + }, + { + "algorithm": "SHA256", + "checksumValue": "de7444c345caa4c47902c4380500356a3ee7e199d2aab84fd8c4960410154f3d" + } + ], + "fileName": "Modules/_hacl/include/krml/types.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "5dd4ee3c835a0d176a6e9fecbe9752fd1474ff41" + }, + { + "algorithm": "SHA256", + "checksumValue": "d82ef594cba44203576d67b047240316bb3c542912ebb7034afa1e07888cec56" + } + ], + "fileName": "Modules/_hacl/internal/Hacl_Hash_MD5.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "515b3082eb7c30597773e1c63ec46688f6da3634" + }, + { + "algorithm": "SHA256", + "checksumValue": "10aacf847006b8e0dfb64d5c327443f954db6718b4aec712fb3268230df6a752" + } + ], + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA1.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "a044ec12b70ba97b67e9a312827d6270452a20ca" + }, + { + "algorithm": "SHA256", + "checksumValue": "a1426b54fa7273ba5b50817c25b2b26fc85c4d1befb14092cd27dc4c99439463" + } + ], + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cfb7b520c39a73cb84c541d370455f92b998781f" + }, + { + "algorithm": "SHA256", + "checksumValue": "fd41997f9e96b3c9a3337b1b51fab965a1e21b0c16f353d156f1a1fa00709fbf" + } + ], + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA3.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f5c7b3ed911af6c8d582e8b3714b0c36195dc994" + }, + { + "algorithm": "SHA256", + "checksumValue": "07de72398b12957e014e97b9ac197bceef12d6d6505c2bfe8b23ee17b94ec5fa" + } + ], + "fileName": "Modules/_hacl/python_hacl_namespaces.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2-config.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ff5e3ae2360adf7279a9c54d12a1d32e16a1f223" + }, + { + "algorithm": "SHA256", + "checksumValue": "1eb919e885244e43cdf7b2104ad30dc9271513478c0026f6bfb4bad6e2f0ab42" + } + ], + "fileName": "Modules/_blake2/impl/blake2-config.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2-impl.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "28b947b43bdc680b9f4335712bb2a5f2d5d32623" + }, + { + "algorithm": "SHA256", + "checksumValue": "4277092643b289f1d36d32cf0fd2efc30ead8bdd99342e5da3b3609dd8ea7d86" + } + ], + "fileName": "Modules/_blake2/impl/blake2-impl.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "caa3da7953109d0d2961e3b686d2d285c484b901" + }, + { + "algorithm": "SHA256", + "checksumValue": "2f6c9d0ecf70be474f2853b52394993625a32960e0a64eae147ef97a3a5c1460" + } + ], + "fileName": "Modules/_blake2/impl/blake2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "029a98f87a178936d9e5211c7798b3e0fc622f94" + }, + { + "algorithm": "SHA256", + "checksumValue": "b392a6e7b43813a05609e994db5fc3552c5912bd482efc781daa0778eb56ab4e" + } + ], + "fileName": "Modules/_blake2/impl/blake2b-load-sse2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse41.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "fb466dd72344170d09e311e5ea12de99ce071357" + }, + { + "algorithm": "SHA256", + "checksumValue": "cc3072c92164142bf2f9dda4e6c08db61be68ec15a95442415e861090d08f6a2" + } + ], + "fileName": "Modules/_blake2/impl/blake2b-load-sse41.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-ref.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4c0d79128cf891a95b1f668031d55c0c6d2e0270" + }, + { + "algorithm": "SHA256", + "checksumValue": "07b257d44e9cc2d95d4911629c92138feafd16d63fef0a5fa7b38914dfd82349" + } + ], + "fileName": "Modules/_blake2/impl/blake2b-ref.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-round.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4c7418e2026417c9c6736fcd305a31f23e05a661" + }, + { + "algorithm": "SHA256", + "checksumValue": "fa34a60c2d198a0585033f43fd4003f4ba279c9ebcabdf5d6650def0e6d1e914" + } + ], + "fileName": "Modules/_blake2/impl/blake2b-round.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "6fa074693aa7305018dfa8db48010a8ef1050ad4" + }, + { + "algorithm": "SHA256", + "checksumValue": "c8c6dd861ac193d4a0e836242ff44900f83423f86d2c2940c8c4c1e41fbd5812" + } + ], + "fileName": "Modules/_blake2/impl/blake2b.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ad3f79b6cbe3fd812722114a0d5d08064e69e4d0" + }, + { + "algorithm": "SHA256", + "checksumValue": "57f1ac6c09f4a50d95811529062220eab4f29cec3805bc6081dec00426c6df62" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-load-sse2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse41.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "51c32d79f419f3d2eb9875cd9a7f5c0d7892f8a8" + }, + { + "algorithm": "SHA256", + "checksumValue": "ecc9e09adcbe098629eafd305596bed8d7004be1d83f326995def42bbde93b23" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-load-sse41.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-xop.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2749a7ba0104b765d4f56f13faf70b6eb89cf203" + }, + { + "algorithm": "SHA256", + "checksumValue": "8bc95595cec4c50f5d70f2b330d3798de07cc784e8890791b3328890e602d5c5" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-load-xop.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-ref.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "883fcfe85f9063819f21b1100296d1f9eb55bac1" + }, + { + "algorithm": "SHA256", + "checksumValue": "9715c00d0f11587a139b07fa26678e6d26e44d3d4910b96158d158da2b022bfb" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-ref.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-round.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "5d9f69adda40ed163b287b9ed4cedb35b88f2daa" + }, + { + "algorithm": "SHA256", + "checksumValue": "65d90111c89c43bb18a9e1d1a4fdbd9f85bebd1ff00129335b85995d0f30ee8b" + } + ], + "fileName": "Modules/_blake2/impl/blake2s-round.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d2691353fa54ac6ffcd7c0a294984dc9d7968ef7" + }, + { + "algorithm": "SHA256", + "checksumValue": "cfd7948c9fd50e9f9c62f8a93b20a254d1d510a862d1092af4f187b7c1a859a3" + } + ], + "fileName": "Modules/_blake2/impl/blake2s.c" + }, + { + "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-init-.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0fbc026a9771d9675e7094790b5b945334d3cb53" + }, + { + "algorithm": "SHA256", + "checksumValue": "1e77c01eec8f167ed10b754f153c0c743c8e5196ae9c81dffc08f129ab56dbfd" + } + ], + "fileName": "Lib/ctypes/macholib/__init__.py" + }, + { + "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-dyld.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4a78ebd73ce4167c722689781a15fe0b4578e967" + }, + { + "algorithm": "SHA256", + "checksumValue": "eb8e7b17f1533bc3e86e23e8695f7a5e4b7a99ef1b1575d10af54f389161b655" + } + ], + "fileName": "Lib/ctypes/macholib/dyld.py" + }, + { + "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-dylib.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f339420cc01bd01f8d0da19b6102f099075e8bcd" + }, + { + "algorithm": "SHA256", + "checksumValue": "f19ee056b18165cc6735efab0b4ca3508be9405b9646c38113316c15e8278a6f" + } + ], + "fileName": "Lib/ctypes/macholib/dylib.py" + }, + { + "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-framework.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0b219f58467d7f193fa1de0c1b118485840d855b" + }, + { + "algorithm": "SHA256", + "checksumValue": "302439e40d9cbdd61b8b7cffd0b7e1278a6811b635044ee366a36e0d991f62da" + } + ], + "fileName": "Lib/ctypes/macholib/framework.py" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-README.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "bda6e0bd6121f7069b420bdc0bc7c49414d948d1" + }, + { + "algorithm": "SHA256", + "checksumValue": "89926cd0fe6cfb33a2b5b7416c101e9b5d42b0d639d348e0871acf6ffc8258a3" + } + ], + "fileName": "Modules/_decimal/libmpdec/README.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "33757ce2ec0c93c1b5e03c45a495563a00e498ae" + }, + { + "algorithm": "SHA256", + "checksumValue": "ad498362c31a5b99ab19fce320ac540cf14c5c4ec09478f0ad3858da1428113d" + } + ], + "fileName": "Modules/_decimal/libmpdec/basearith.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "bf03919412c068e6969e7ac48850f91bfcd3b2b1" + }, + { + "algorithm": "SHA256", + "checksumValue": "2eaac88a71b9bcf3144396c12dcfeced573e0e550a0050d75b9ed3903248596d" + } + ], + "fileName": "Modules/_decimal/libmpdec/basearith.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-bench.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "c925b7f26754ae182aaa461d51802e8b6a2bb5e9" + }, + { + "algorithm": "SHA256", + "checksumValue": "007e38542ec8d9d8805fe243b5390d79211b9360e2797a20079e833e68ad9e45" + } + ], + "fileName": "Modules/_decimal/libmpdec/bench.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-bench-full.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cb22686269685a53a17afdea9ed984714e399d9d" + }, + { + "algorithm": "SHA256", + "checksumValue": "1b9e892d4b268deea835ec8906f20a1e5d25e037b2e698edcd34315613f3608c" + } + ], + "fileName": "Modules/_decimal/libmpdec/bench_full.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-bits.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "fc91c2450cdf1e785d1347411662294c3945eb27" + }, + { + "algorithm": "SHA256", + "checksumValue": "ce7741e58ea761a24250c0bfa10058cec8c4fd220dca70a41de3927a2e4f5376" + } + ], + "fileName": "Modules/_decimal/libmpdec/bits.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "7187c18916b0a546ec19b4fc4bec43d0d9fb5fc2" + }, + { + "algorithm": "SHA256", + "checksumValue": "cd430b8657cf8a616916e02f9bd5ca044d5fc19e69333f5d427e1fdb90b0864b" + } + ], + "fileName": "Modules/_decimal/libmpdec/constants.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "af9cbd016fb0ef0b30ced49c0aa4ce2ca3c20125" + }, + { + "algorithm": "SHA256", + "checksumValue": "19dc46df04abb7ee08e9a403f87c8aac8d4a077efcce314c597f8b73e22884f2" + } + ], + "fileName": "Modules/_decimal/libmpdec/constants.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-context.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "666162870230bebd3f2383020d908806fd03909e" + }, + { + "algorithm": "SHA256", + "checksumValue": "9a265d366f31894aad78bca7fcdc1457bc4a3aa3887ca231b7d78e41f79541c0" + } + ], + "fileName": "Modules/_decimal/libmpdec/context.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0545547a8b37b922fbe2574fbad8fc3bf16f1d33" + }, + { + "algorithm": "SHA256", + "checksumValue": "66fe27b9bb37039cad5be32b105ed509e5aefa15c1957a9058af8ee23cddc97a" + } + ], + "fileName": "Modules/_decimal/libmpdec/convolute.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "05ff0936c5bb08f40d460f5843004a1cc0751d9b" + }, + { + "algorithm": "SHA256", + "checksumValue": "c00d17450c2b8e1d7f1eb8a084f7e6a68f257a453f8701600e860bf357c531d7" + } + ], + "fileName": "Modules/_decimal/libmpdec/convolute.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "fe8176849bc99a306332ba25caa4e91bfa3c6f7d" + }, + { + "algorithm": "SHA256", + "checksumValue": "1f4e65c44864c3e911a6e91f33adec76765293e90553459e3ebce35a58898dba" + } + ], + "fileName": "Modules/_decimal/libmpdec/crt.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "1930b9e0910014b3479aec4e940f02118d9e4a08" + }, + { + "algorithm": "SHA256", + "checksumValue": "7d31f1d0dd73b62964dab0f7a1724473bf87f1f95d8febf0b40c15430ae9a47c" + } + ], + "fileName": "Modules/_decimal/libmpdec/crt.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "415c51e7d7f517b6366bec2a809610d0d38ada14" + }, + { + "algorithm": "SHA256", + "checksumValue": "0a9fef8a374f55277e9f6000b7277bb037b9763c32b156c29950422b057498bd" + } + ], + "fileName": "Modules/_decimal/libmpdec/difradix2.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d8a998c3bee4c3d9059ba7bf9ae6a8b64649c2ba" + }, + { + "algorithm": "SHA256", + "checksumValue": "5c6766496224de657400995b58b64db3e7084004bf00daebdd7e08d0c5995243" + } + ], + "fileName": "Modules/_decimal/libmpdec/difradix2.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-README.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "158f6ad18edf348efa4fdd7cf61114c77c1d22e9" + }, + { + "algorithm": "SHA256", + "checksumValue": "7b0da2758097a2688f06b3c7ca46b2ebc8329addbd28bb4f5fe95626cc81f8a9" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/README.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-compare.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ef80ba26847287fb351ab0df0a78b5f08ba0b5b7" + }, + { + "algorithm": "SHA256", + "checksumValue": "452666ee4eb10a8cf0a926cb3bcf5e95b5c361fa129dbdfe27b654e6d640417e" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/compare.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-div.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "6ca3a369b3d1e140fdc93c4fdbedb724f7daf969" + }, + { + "algorithm": "SHA256", + "checksumValue": "6d369f5a24d0bb1e7cb6a4f8b0e97a273260e7668c8a540a8fcc92e039f7af2e" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/div.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-divmod.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "3872a28b4f77e07e1760256067ea338a8dd183f8" + }, + { + "algorithm": "SHA256", + "checksumValue": "5db54bae75ac3d7fa12f1bb0f7ce1bf797df86a81030e8c3ce44d3b1f9b958b7" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/divmod.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-multiply.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "25dbc94fd4ee5dec21061d2d40dd5d0f88849cb1" + }, + { + "algorithm": "SHA256", + "checksumValue": "22ed39b18fa740a27aacfd29a7bb40066be24500ba49b9b1f24e2af1e039fcd9" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/multiply.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-pow.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "13d3b7657dc2dc5000fea428f57963d520792ef7" + }, + { + "algorithm": "SHA256", + "checksumValue": "cd8c037649b3d4d6897c9acd2f92f3f9d5390433061d5e48623a5d526a3f4f9c" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/pow.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-powmod.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "1f7e6c3d3e38df52bbcec0f5a180a8f328679618" + }, + { + "algorithm": "SHA256", + "checksumValue": "e29614b43abf1856b656a84d6b67c22cc5dc7af8cbae8ddc7acf17022220ee12" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/powmod.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-shift.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0bd9ce89c7987d1109eb7b0c8f1f9a1298e1422e" + }, + { + "algorithm": "SHA256", + "checksumValue": "203f2dbf11d115580cb3c7c524ac6ccca2a7b31d89545db1b6263381b5de2b6a" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/shift.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-sqrt.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b401ba0814e17c9164c0df26e01cc0a355382f46" + }, + { + "algorithm": "SHA256", + "checksumValue": "f3dc2ce321833bbd4b3d1d9ea6fa2e0bcc1bfe1e39abb8d55be53e46c33949db" + } + ], + "fileName": "Modules/_decimal/libmpdec/examples/sqrt.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "060615ddef089a5a8f879a57e4968d920972a0e2" + }, + { + "algorithm": "SHA256", + "checksumValue": "a9f923524d53a9445769f27405375ec3d95fa804bb11db5ee249ae047f11cfce" + } + ], + "fileName": "Modules/_decimal/libmpdec/fnt.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b205043ebeaf065b16505a299342a992654f19b0" + }, + { + "algorithm": "SHA256", + "checksumValue": "3b03e69adf78fde68c8f87d33595d557237581d33fc067e1039eed9e9f2cc44c" + } + ], + "fileName": "Modules/_decimal/libmpdec/fnt.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "702c27599b43280c94906235d7e1a74193ba701b" + }, + { + "algorithm": "SHA256", + "checksumValue": "cf2e69b946ec14b087e523c0ff606553070d13c23e851fb0ba1df51a728017e6" + } + ], + "fileName": "Modules/_decimal/libmpdec/fourstep.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "ee5291c265ef1f5ae373bc243a4d96975eb3e7b5" + }, + { + "algorithm": "SHA256", + "checksumValue": "dbaced03b52d0f880c377b86c943bcb36f24d557c99a5e9732df3ad5debb5917" + } + ], + "fileName": "Modules/_decimal/libmpdec/fourstep.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-io.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "12402bcf7f0161adb83f78163f41cc10a5e5de5f" + }, + { + "algorithm": "SHA256", + "checksumValue": "cba044c76b6bc3ae6cfa49df1121cad7552140157b9e61e11cbb6580cc5d74cf" + } + ], + "fileName": "Modules/_decimal/libmpdec/io.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-io.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "28c653cd40b1ce46575e41f5dbfda5f6dd0db4d1" + }, + { + "algorithm": "SHA256", + "checksumValue": "259eab89fe27914e0e39e61199094a357ac60d86b2aab613c909040ff64a4a0c" + } + ], + "fileName": "Modules/_decimal/libmpdec/io.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-REFERENCES.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "218d1d7bedb335cd2c31eae89a15873c3139e13f" + }, + { + "algorithm": "SHA256", + "checksumValue": "a57e8bed93ded481ef264166aec2c49d1a7f3252f29a873ee41fff053cfd9c20" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/REFERENCES.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-bignum.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f67eab2431336cf6eeafb30cdafd7e54c251def3" + }, + { + "algorithm": "SHA256", + "checksumValue": "dc34aa122c208ce79e3fc6baee8628094ffaf6a662862dd5647836241f6ebd79" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/bignum.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-fnt.py", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "a58cfbcd8ea57d66ddfd11fb5a170138c8bbfb3a" + }, + { + "algorithm": "SHA256", + "checksumValue": "122de20eebf87274af2d02072251a94500e7df2d5ef29e81aeabeda991c079e3" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/fnt.py" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-matrix-transform.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "9a947f6b660150cbd457c4458da2956a36c5824d" + }, + { + "algorithm": "SHA256", + "checksumValue": "592659e7192e3a939b797f5bc7455455834a285f5d8b643ccd780b5114914f73" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/matrix-transform.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-64.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "69fe9afb8353b5a2b57917469c51c64ac518169d" + }, + { + "algorithm": "SHA256", + "checksumValue": "229a80ca940c594a32e3345412370cbc097043fe59c66a6153cbcf01e7837266" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/mulmod-64.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-ppro.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "720d468a1f51098036c7a0c869810fff22ed9b79" + }, + { + "algorithm": "SHA256", + "checksumValue": "f3549fc73f697a087267c7b042e30a409e191cbba69a2c0902685e507fbae9f7" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/mulmod-ppro.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-six-step.txt", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "6815ec3a39baebebe7b3f51d45d10c180a659f17" + }, + { + "algorithm": "SHA256", + "checksumValue": "bf15f73910a173c98fca9db56122b6cc71983668fa8b934c46ca21a57398ec54" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/six-step.txt" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-umodarith.lisp", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "c91ac4438e661ce78f86e981257546e5adff39ae" + }, + { + "algorithm": "SHA256", + "checksumValue": "783a1b4b9b7143677b0c3d30ffaf28aa0cb01956409031fa38ed8011970bdee0" + } + ], + "fileName": "Modules/_decimal/libmpdec/literature/umodarith.lisp" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "7e8dfb4b7a801b48c501969b001153203b14679e" + }, + { + "algorithm": "SHA256", + "checksumValue": "5ba2f4c80302e71fb216aa247c858e0bf6c8cfabffe7980ac17d4d023c0fef2b" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpalloc.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "bccb6a6ae76fd7f6c8a9102a78958bcad7862950" + }, + { + "algorithm": "SHA256", + "checksumValue": "f7412521de38afb837fcabc2b1d48b971b86bfaa55f3f40d58ff9e46e92debd3" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpalloc.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f4539afb1ace58c52d18ffd0cc7704f53ca55182" + }, + { + "algorithm": "SHA256", + "checksumValue": "4f89b8095e408a18deff79cfb605299e615bae747898eb105d8936064f7fb626" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpdecimal.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4b80e25ac49b7e1ea0d1e84967ee32a3d111fc4c" + }, + { + "algorithm": "SHA256", + "checksumValue": "ea0b9c6b296c13aed6ecaa50b463e39a9c1bdc059b84f50507fd8247b2e660f9" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpdecimal.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpsignal.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "5c7305a6db0fddf64c6d97e29d3b0c402e3d5d6e" + }, + { + "algorithm": "SHA256", + "checksumValue": "653171cf2549719478417db7e9800fa0f9d99c02dec6da6876329ccf2c07b93f" + } + ], + "fileName": "Modules/_decimal/libmpdec/mpsignal.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d736b874c43777ca021dde5289ea718893f39219" + }, + { + "algorithm": "SHA256", + "checksumValue": "bdbf2e246f341a3ba3f6f9d8759e7cb222eb9b15f9ed1e7c9f6a59cbb9f8bc91" + } + ], + "fileName": "Modules/_decimal/libmpdec/numbertheory.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d341508d8c6dd4c4cbd8b99afc8029945f9bbe0d" + }, + { + "algorithm": "SHA256", + "checksumValue": "2f7d5b40af508fa6ac86f5d62101fa3bf683c63b24aa87c9548e3fdd13abc57b" + } + ], + "fileName": "Modules/_decimal/libmpdec/numbertheory.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cbd05d68bb3940d0d7d0818b14cc03b090a4dd74" + }, + { + "algorithm": "SHA256", + "checksumValue": "7602aaf98ec9525bc4b3cab9631615e1be2efd9af894002ef4e3f5ec63924fcf" + } + ], + "fileName": "Modules/_decimal/libmpdec/sixstep.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "4c059463ec4b4522562dab24760fc64c172c9eee" + }, + { + "algorithm": "SHA256", + "checksumValue": "a191366348b3d3dd49b9090ec5c77dbd77bb3a523c01ff32adafa137e5097ce7" + } + ], + "fileName": "Modules/_decimal/libmpdec/sixstep.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "cc5593ac9fdb60480cc23fc9d6f27d85670bd35f" + }, + { + "algorithm": "SHA256", + "checksumValue": "2d12fcae512143a9376c8a0d4c1ba3008e420e024497a7e7ec64c6bec23fcddc" + } + ], + "fileName": "Modules/_decimal/libmpdec/transpose.c" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2f616425756b6cbdf7d189744870b98b613455bd" + }, + { + "algorithm": "SHA256", + "checksumValue": "fafeb2b901b2b41bf0df00be7d99b84df1a78e3cc1e582e09cbfc3b6d44d4abe" + } + ], + "fileName": "Modules/_decimal/libmpdec/transpose.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-typearith.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "b1e9341e173cc8e219ad4aa45fad36d92cce10d3" + }, + { + "algorithm": "SHA256", + "checksumValue": "25e0a0703b51744277834e6b2398d7b7d2c17f92bf30f8b6f949e0486ae2b346" + } + ], + "fileName": "Modules/_decimal/libmpdec/typearith.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-umodarith.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "46f6483fce136cd3cc2f7516ee119a487d86333e" + }, + { + "algorithm": "SHA256", + "checksumValue": "bfe1ddb2ca92906456b80745adcbe02c83cadac3ef69caa21bc09b7292cc152b" + } + ], + "fileName": "Modules/_decimal/libmpdec/umodarith.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-vcdiv64.asm", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "d0cc1052fcba08b773d935b0ae2dc6b80d0f2f68" + }, + { + "algorithm": "SHA256", + "checksumValue": "aacc3e47ea8f41e8840c6c67f64ec96d54696a16889903098fa1aab56949a00f" + } + ], + "fileName": "Modules/_decimal/libmpdec/vcdiv64.asm" + }, + { + "SPDXID": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-23.3.2-py3-none-any.whl", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "8e48f55ab2965ee64bd55cc91a8077d184a33e30" + }, + { + "algorithm": "SHA256", + "checksumValue": "5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76" + } + ], + "fileName": "Lib/ensurepip/_bundled/pip-23.3.2-py3-none-any.whl" + } + ], + "packages": [ + { + "SPDXID": "SPDXRef-PACKAGE-expat", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "6b902ab103843592be5e99504f846ec109c1abb692e85347587f237a4ffa1033" + } + ], + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.5.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "MIT", + "name": "expat", + "originator": "Organization: Expat development team", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.5.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-hacl-star", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "c23ac158b238c368389dc86bfc315263e5c0e57785da74144aea2cab9a3d51a2" + } + ], + "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/521af282fdf6d60227335120f18ae9309a4b8e8c.zip", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:521af282fdf6d60227335120f18ae9309a4b8e8c:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "Apache-2.0", + "name": "hacl-star", + "originator": "Organization: HACL* Developers", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "521af282fdf6d60227335120f18ae9309a4b8e8c" + }, + { + "SPDXID": "SPDXRef-PACKAGE-libb2", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "53626fddce753c454a3fea581cbbc7fe9bbcf0bc70416d48fdbbf5d87ef6c72e" + } + ], + "downloadLocation": "https://github.com/BLAKE2/libb2/releases/download/v0.98.1/libb2-0.98.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:blake2:libb2:0.98.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "CC0-1.0", + "name": "libb2", + "originator": "Organization: BLAKE2 - fast secure hashing", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.98.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-macholib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "c76f268f5054024e962f2515a0e522baf85313064f6740d80375afc850787a38" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/ec/57/f0a712efc3ed982cf4038a3cee172057303b9be914c32c93b2fbec27f785/macholib-1.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/macholib@1.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "macholib", + "originator": "Person: Ronald Oussoren (ronaldoussoren@mac.com)", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-mpdecimal", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "9f9cd4c041f99b5c49ffb7b59d9f12d95b683d88585608aa56a6307667b2b21f" + } + ], + "downloadLocation": "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:2.5.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "BSD-2-Clause", + "name": "mpdecimal", + "originator": "Organization: bytereef.org", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.5.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pip", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/15/aa/3f4c7bcee2057a76562a5b33ecbd199be08cdb4443a02e26bd2c3cf6fc39/pip-23.3.2-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:pypa:pip:23.3.2:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + }, + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pip@23.3.2", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "pip", + "originator": "Organization: Python Packaging Authority", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "23.3.2" + } + ], + "relationships": [ + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-ascii.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-asciitab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat-config.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-expat-external.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-iasciitab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-internal.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-latin1tab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-nametab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-pyexpatns.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-siphash.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-utf8tab.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-winconfig.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlparse.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlrole.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlrole.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-impl.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-impl.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmltok-ns.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2-config.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2-impl.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse41.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-ref.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-round.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse41.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-xop.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-ref.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-round.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-libb2" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-init-.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-macholib" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-dyld.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-macholib" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-dylib.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-macholib" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-framework.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-macholib" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-README.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-bench.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-bench-full.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-bits.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-context.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-README.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-compare.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-div.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-divmod.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-multiply.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-pow.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-powmod.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-shift.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-sqrt.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-io.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-io.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-REFERENCES.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-bignum.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-fnt.py", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-matrix-transform.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-64.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-ppro.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-six-step.txt", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-umodarith.lisp", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpsignal.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-typearith.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-umodarith.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-vcdiv64.asm", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Lib-ensurepip-bundled-pip-23.3.2-py3-none-any.whl", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-pip" + } + ], + "spdxVersion": "SPDX-2.3" +} \ No newline at end of file diff --git a/Modules/Setup b/Modules/Setup index 1367f0ef4fa54a..8ad9a5aebbfcaa 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -273,6 +273,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_xxsubinterpreters _xxsubinterpretersmodule.c #_xxinterpchannels _xxinterpchannelsmodule.c +#_xxinterpqueues _xxinterpqueuesmodule.c #_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c #_testbuffer _testbuffer.c #_testinternalcapi _testinternalcapi.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 54650ea9c1d4ac..8a65a9cffb1b9d 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -41,8 +41,11 @@ @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c @MODULE__STRUCT_TRUE@_struct _struct.c + +# build supports subinterpreters @MODULE__XXSUBINTERPRETERS_TRUE@_xxsubinterpreters _xxsubinterpretersmodule.c @MODULE__XXINTERPCHANNELS_TRUE@_xxinterpchannels _xxinterpchannelsmodule.c +@MODULE__XXINTERPQUEUES_TRUE@_xxinterpqueues _xxinterpqueuesmodule.c @MODULE__ZONEINFO_TRUE@_zoneinfo _zoneinfo.c # needs libm diff --git a/Modules/_csv.c b/Modules/_csv.c index 714fbef08d22c9..ae6b6457ffad9a 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -160,15 +160,9 @@ static PyObject * get_dialect_from_registry(PyObject *name_obj, _csvstate *module_state) { PyObject *dialect_obj; - - dialect_obj = PyDict_GetItemWithError(module_state->dialects, name_obj); - if (dialect_obj == NULL) { - if (!PyErr_Occurred()) - PyErr_Format(module_state->error_obj, "unknown dialect"); + if (PyDict_GetItemRef(module_state->dialects, name_obj, &dialect_obj) == 0) { + PyErr_SetString(module_state->error_obj, "unknown dialect"); } - else - Py_INCREF(dialect_obj); - return dialect_obj; } diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index f909a9496b6526..fc16b9176fd1c0 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4354,10 +4354,10 @@ _init_pos_args(PyObject *self, PyTypeObject *type, return index; } - for (i = 0; - i < dict->length && (i+index) < PyTuple_GET_SIZE(args); + for (i = index; + i < dict->length && i < PyTuple_GET_SIZE(args); ++i) { - PyObject *pair = PySequence_GetItem(fields, i); + PyObject *pair = PySequence_GetItem(fields, i - index); PyObject *name, *val; int res; if (!pair) @@ -4367,7 +4367,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, Py_DECREF(pair); return -1; } - val = PyTuple_GET_ITEM(args, i + index); + val = PyTuple_GET_ITEM(args, i); if (kwds) { res = PyDict_Contains(kwds, name); if (res != 0) { @@ -4388,7 +4388,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, if (res == -1) return -1; } - return index + dict->length; + return dict->length; } static int diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index d33e6fc7586d28..ecc60417790417 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,6 +1,4 @@ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED @@ -98,11 +96,10 @@ typedef struct { } Test2; EXPORT(int) -_testfunc_array_in_struct1(Test2 in) +_testfunc_array_in_struct2(Test2 in) { int result = 0; - - for (unsigned i = 0; i < 16; i++) + for (unsigned i = 0; i < sizeof(in.data)/sizeof(in.data[0]); i++) result += in.data[i]; /* As the structure is passed by value, changes to it shouldn't be * reflected in the caller. @@ -111,22 +108,25 @@ _testfunc_array_in_struct1(Test2 in) return result; } -typedef struct { - double data[2]; -} Test3; - +/* + * Test3A struct test the MAX_STRUCT_SIZE 16 with single precision floats. + * These structs should be passed via registers on all platforms and they are + * used for within bounds tests. + */ typedef struct { float data[2]; float more_data[2]; -} Test3B; +} Test3A; EXPORT(double) -_testfunc_array_in_struct2(Test3 in) +_testfunc_array_in_struct3A(Test3A in) { double result = 0; - for (unsigned i = 0; i < 2; i++) + for (unsigned i = 0; i < sizeof(in.data)/sizeof(in.data[0]); i++) result += in.data[i]; + for (unsigned i = 0; i < sizeof(in.more_data)/sizeof(in.more_data[0]); i++) + result += in.more_data[i]; /* As the structure is passed by value, changes to it shouldn't be * reflected in the caller. */ @@ -134,20 +134,116 @@ _testfunc_array_in_struct2(Test3 in) return result; } +/* The structs Test3B..Test3E use the same functions hence using the MACRO + * to define their implementation. + */ + +#define _TESTFUNC_ARRAY_IN_STRUCT_IMPL \ + double result = 0; \ + \ + for (unsigned i = 0; i < sizeof(in.data)/sizeof(in.data[0]); i++) \ + result += in.data[i]; \ + /* As the structure is passed by value, changes to it shouldn't be \ + * reflected in the caller. \ + */ \ + memset(in.data, 0, sizeof(in.data)); \ + return result; \ + +#define _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL \ + for (unsigned i = 0; i < sizeof(s.data)/sizeof(s.data[0]); i++) \ + s.data[i] = (double)i+1; \ + return s; \ + + +/* + * Test3B struct test the MAX_STRUCT_SIZE 16 with double precision floats. + * These structs should be passed via registers on all platforms and they are + * used for within bounds tests. + */ +typedef struct { + double data[2]; +} Test3B; + EXPORT(double) -_testfunc_array_in_struct2a(Test3B in) +_testfunc_array_in_struct3B(Test3B in) { - double result = 0; + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} - for (unsigned i = 0; i < 2; i++) - result += in.data[i]; - for (unsigned i = 0; i < 2; i++) - result += in.more_data[i]; - /* As the structure is passed by value, changes to it shouldn't be - * reflected in the caller. - */ - memset(in.data, 0, sizeof(in.data)); - return result; +EXPORT(Test3B) +_testfunc_array_in_struct3B_set_defaults(void) +{ + Test3B s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL +} + +/* + * Test3C struct tests the MAX_STRUCT_SIZE 32. Structs containing arrays of up + * to four floating point types are passed in registers on Arm platforms. + * This struct is used for within bounds test on Arm platfroms and for an + * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 32. + * See gh-110190. + */ +typedef struct { + double data[4]; +} Test3C; + +EXPORT(double) +_testfunc_array_in_struct3C(Test3C in) +{ + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} + +EXPORT(Test3C) +_testfunc_array_in_struct3C_set_defaults(void) +{ + Test3C s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL +} + +/* + * Test3D struct tests the MAX_STRUCT_SIZE 64. Structs containing arrays of up + * to eight floating point types are passed in registers on PPC64LE platforms. + * This struct is used for within bounds test on PPC64LE platfroms and for an + * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 64. + * See gh-110190. + */ +typedef struct { + double data[8]; +} Test3D; + +EXPORT(double) +_testfunc_array_in_struct3D(Test3D in) +{ + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} + +EXPORT(Test3D) +_testfunc_array_in_struct3D_set_defaults(void) +{ + Test3D s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL +} + +/* + * Test3E struct tests the out-of-bounds for all architectures. + * See gh-110190. + */ +typedef struct { + double data[9]; +} Test3E; + +EXPORT(double) +_testfunc_array_in_struct3E(Test3E in) +{ + _TESTFUNC_ARRAY_IN_STRUCT_IMPL +} + +EXPORT(Test3E) +_testfunc_array_in_struct3E_set_defaults(void) +{ + Test3E s; + _TESTFUNC_ARRAY_IN_STRUCT_SET_DEFAULTS_IMPL } typedef union { diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 6fbcf77a115371..fb3e20e8db3e27 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -695,31 +695,49 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct stgdict->size = aligned_size; stgdict->align = total_align; - stgdict->length = len; /* ADD ffi_ofs? */ + stgdict->length = ffi_ofs + len; -#define MAX_STRUCT_SIZE 16 +/* + * The value of MAX_STRUCT_SIZE depends on the platform Python is running on. + */ +#if defined(__aarch64__) || defined(__arm__) +# define MAX_STRUCT_SIZE 32 +#elif defined(__powerpc64__) +# define MAX_STRUCT_SIZE 64 +#else +# define MAX_STRUCT_SIZE 16 +#endif if (arrays_seen && (size <= MAX_STRUCT_SIZE)) { /* - * See bpo-22273. Arrays are normally treated as pointers, which is - * fine when an array name is being passed as parameter, but not when - * passing structures by value that contain arrays. On 64-bit Linux, - * small structures passed by value are passed in registers, and in + * See bpo-22273 and gh-110190. Arrays are normally treated as + * pointers, which is fine when an array name is being passed as + * parameter, but not when passing structures by value that contain + * arrays. + * Small structures passed by value are passed in registers, and in * order to do this, libffi needs to know the true type of the array * members of structs. Treating them as pointers breaks things. * - * By small structures, we mean ones that are 16 bytes or less. In that - * case, there can't be more than 16 elements after unrolling arrays, - * as we (will) disallow bitfields. So we can collect the true ffi_type - * values in a fixed-size local array on the stack and, if any arrays - * were seen, replace the ffi_type_pointer.elements with a more - * accurate set, to allow libffi to marshal them into registers - * correctly. It means one more loop over the fields, but if we got - * here, the structure is small, so there aren't too many of those. + * Small structures have different sizes depending on the platform + * where Python is running on: * - * Although the passing in registers is specific to 64-bit Linux, the - * array-in-struct vs. pointer problem is general. But we restrict the - * type transformation to small structs nonetheless. + * * x86-64: 16 bytes or less + * * Arm platforms (both 32 and 64 bit): 32 bytes or less + * * PowerPC 64 Little Endian: 64 bytes or less + * + * In that case, there can't be more than 16, 32 or 64 elements after + * unrolling arrays, as we (will) disallow bitfields. + * So we can collect the true ffi_type values in a fixed-size local + * array on the stack and, if any arrays were seen, replace the + * ffi_type_pointer.elements with a more accurate set, to allow + * libffi to marshal them into registers correctly. + * It means one more loop over the fields, but if we got here, + * the structure is small, so there aren't too many of those. + * + * Although the passing in registers is specific to the above + * platforms, the array-in-struct vs. pointer problem is general. + * But we restrict the type transformation to small structs + * nonetheless. * * Note that although a union may be small in terms of memory usage, it * could contain many overlapping declarations of arrays, e.g. @@ -745,6 +763,8 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct * struct { uint_32 e1; uint_32 e2; ... uint_32 e_4; } f6; * } * + * The same principle applies for a struct 32 or 64 bytes in size. + * * So the struct/union needs setting up as follows: all non-array * elements copied across as is, and all array elements replaced with * an equivalent struct which has as many fields as the array has diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f9d5793f9b6497..b574c96d3f9625 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -98,6 +98,7 @@ typedef struct { PyTypeObject *TreeBuilder_Type; PyTypeObject *XMLParser_Type; + PyObject *expat_capsule; struct PyExpat_CAPI *expat_capi; } elementtreestate; @@ -155,6 +156,7 @@ elementtree_clear(PyObject *m) Py_CLEAR(st->ElementIter_Type); Py_CLEAR(st->TreeBuilder_Type); Py_CLEAR(st->XMLParser_Type); + Py_CLEAR(st->expat_capsule); st->expat_capi = NULL; return 0; @@ -175,6 +177,7 @@ elementtree_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->ElementIter_Type); Py_VISIT(st->TreeBuilder_Type); Py_VISIT(st->XMLParser_Type); + Py_VISIT(st->expat_capsule); return 0; } @@ -3066,6 +3069,7 @@ typedef struct { PyObject *handle_close; elementtreestate *state; + PyObject *elementtree_module; } XMLParserObject; /* helpers */ @@ -3607,7 +3611,11 @@ xmlparser_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->handle_start = self->handle_data = self->handle_end = NULL; self->handle_comment = self->handle_pi = self->handle_close = NULL; self->handle_doctype = NULL; - self->state = get_elementtree_state_by_type(type); + self->elementtree_module = PyType_GetModuleByDef(type, &elementtreemodule); + assert(self->elementtree_module != NULL); + Py_INCREF(self->elementtree_module); + // See gh-111784 for explanation why is reference to module needed here. + self->state = get_elementtree_state(self->elementtree_module); } return (PyObject *)self; } @@ -3784,6 +3792,7 @@ xmlparser_gc_clear(XMLParserObject *self) EXPAT(st, ParserFree)(parser); } + Py_CLEAR(self->elementtree_module); Py_CLEAR(self->handle_close); Py_CLEAR(self->handle_pi); Py_CLEAR(self->handle_comment); @@ -4343,7 +4352,10 @@ module_exec(PyObject *m) goto error; /* link against pyexpat */ - st->expat_capi = PyCapsule_Import(PyExpat_CAPSULE_NAME, 0); + if (!(st->expat_capsule = _PyImport_GetModuleAttrString("pyexpat", "expat_CAPI"))) + goto error; + if (!(st->expat_capi = PyCapsule_GetPointer(st->expat_capsule, PyExpat_CAPSULE_NAME))) + goto error; if (st->expat_capi) { /* check that it's usable */ if (strcmp(st->expat_capi->magic, PyExpat_CAPI_MAGIC) != 0 || @@ -4418,9 +4430,7 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 4f3786676d131f..f02207ace9f3d2 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -10,7 +10,6 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytes_Join() #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() @@ -518,25 +517,20 @@ buffered_closed(buffered *self) return closed; } +/*[clinic input] +@critical_section +@getter +_io._Buffered.closed +[clinic start generated code]*/ + static PyObject * -buffered_closed_get_impl(buffered *self, void *context) +_io__Buffered_closed_get_impl(buffered *self) +/*[clinic end generated code: output=f08ce57290703a1a input=18eddefdfe4a3d2f]*/ { CHECK_INITIALIZED(self) return PyObject_GetAttr(self->raw, &_Py_ID(closed)); } -static PyObject * -buffered_closed_get(buffered *self, void *context) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = buffered_closed_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - - return return_value; -} - /*[clinic input] @critical_section _io._Buffered.close @@ -662,44 +656,35 @@ _io__Buffered_writable_impl(buffered *self) return PyObject_CallMethodNoArgs(self->raw, &_Py_ID(writable)); } + +/*[clinic input] +@critical_section +@getter +_io._Buffered.name +[clinic start generated code]*/ + static PyObject * -buffered_name_get_impl(buffered *self, void *context) +_io__Buffered_name_get_impl(buffered *self) +/*[clinic end generated code: output=d2adf384051d3d10 input=6b84a0e6126f545e]*/ { CHECK_INITIALIZED(self) return PyObject_GetAttr(self->raw, &_Py_ID(name)); } -static PyObject * -buffered_name_get(buffered *self, void *context) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = buffered_name_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - - return return_value; -} +/*[clinic input] +@critical_section +@getter +_io._Buffered.mode +[clinic start generated code]*/ static PyObject * -buffered_mode_get_impl(buffered *self, void *context) +_io__Buffered_mode_get_impl(buffered *self) +/*[clinic end generated code: output=0feb205748892fa4 input=0762d5e28542fd8c]*/ { CHECK_INITIALIZED(self) return PyObject_GetAttr(self->raw, &_Py_ID(mode)); } -static PyObject * -buffered_mode_get(buffered *self, void *context) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = buffered_mode_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - - return return_value; -} - /* Lower-level APIs */ /*[clinic input] @@ -2541,9 +2526,9 @@ static PyMemberDef bufferedreader_members[] = { }; static PyGetSetDef bufferedreader_getset[] = { - {"closed", (getter)buffered_closed_get, NULL, NULL}, - {"name", (getter)buffered_name_get, NULL, NULL}, - {"mode", (getter)buffered_mode_get, NULL, NULL}, + _IO__BUFFERED_CLOSED_GETSETDEF + _IO__BUFFERED_NAME_GETSETDEF + _IO__BUFFERED_MODE_GETSETDEF {NULL} }; @@ -2601,9 +2586,9 @@ static PyMemberDef bufferedwriter_members[] = { }; static PyGetSetDef bufferedwriter_getset[] = { - {"closed", (getter)buffered_closed_get, NULL, NULL}, - {"name", (getter)buffered_name_get, NULL, NULL}, - {"mode", (getter)buffered_mode_get, NULL, NULL}, + _IO__BUFFERED_CLOSED_GETSETDEF + _IO__BUFFERED_NAME_GETSETDEF + _IO__BUFFERED_MODE_GETSETDEF {NULL} }; @@ -2719,9 +2704,9 @@ static PyMemberDef bufferedrandom_members[] = { }; static PyGetSetDef bufferedrandom_getset[] = { - {"closed", (getter)buffered_closed_get, NULL, NULL}, - {"name", (getter)buffered_name_get, NULL, NULL}, - {"mode", (getter)buffered_mode_get, NULL, NULL}, + _IO__BUFFERED_CLOSED_GETSETDEF + _IO__BUFFERED_NAME_GETSETDEF + _IO__BUFFERED_MODE_GETSETDEF {NULL} }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 16b8ac600ace79..4a15c8e841f25f 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -990,7 +990,9 @@ static int bytesio_clear(bytesio *self) { Py_CLEAR(self->dict); - Py_CLEAR(self->buf); + if (self->exports == 0) { + Py_CLEAR(self->buf); + } return 0; } @@ -1095,13 +1097,6 @@ bytesiobuf_releasebuffer(bytesiobuf *obj, Py_buffer *view) b->exports--; } -static int -bytesiobuf_clear(bytesiobuf *self) -{ - Py_CLEAR(self->source); - return 0; -} - static int bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) { @@ -1116,7 +1111,7 @@ bytesiobuf_dealloc(bytesiobuf *self) PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - (void)bytesiobuf_clear(self); + Py_CLEAR(self->source); tp->tp_free(self); Py_DECREF(tp); } @@ -1124,7 +1119,6 @@ bytesiobuf_dealloc(bytesiobuf *self) static PyType_Slot bytesiobuf_slots[] = { {Py_tp_dealloc, bytesiobuf_dealloc}, {Py_tp_traverse, bytesiobuf_traverse}, - {Py_tp_clear, bytesiobuf_clear}, // Buffer protocol {Py_bf_getbuffer, bytesiobuf_getbuffer}, diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 20833a10139681..d5bec5f71f5be8 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -327,6 +327,33 @@ _io__Buffered_simple_flush(buffered *self, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(_io__Buffered_closed_HAS_DOCSTR) +# define _io__Buffered_closed_DOCSTR _io__Buffered_closed__doc__ +#else +# define _io__Buffered_closed_DOCSTR NULL +#endif +#if defined(_IO__BUFFERED_CLOSED_GETSETDEF) +# undef _IO__BUFFERED_CLOSED_GETSETDEF +# define _IO__BUFFERED_CLOSED_GETSETDEF {"closed", (getter)_io__Buffered_closed_get, (setter)_io__Buffered_closed_set, _io__Buffered_closed_DOCSTR}, +#else +# define _IO__BUFFERED_CLOSED_GETSETDEF {"closed", (getter)_io__Buffered_closed_get, NULL, _io__Buffered_closed_DOCSTR}, +#endif + +static PyObject * +_io__Buffered_closed_get_impl(buffered *self); + +static PyObject * +_io__Buffered_closed_get(buffered *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_closed_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_io__Buffered_close__doc__, "close($self, /)\n" "--\n" @@ -442,6 +469,60 @@ _io__Buffered_writable(buffered *self, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(_io__Buffered_name_HAS_DOCSTR) +# define _io__Buffered_name_DOCSTR _io__Buffered_name__doc__ +#else +# define _io__Buffered_name_DOCSTR NULL +#endif +#if defined(_IO__BUFFERED_NAME_GETSETDEF) +# undef _IO__BUFFERED_NAME_GETSETDEF +# define _IO__BUFFERED_NAME_GETSETDEF {"name", (getter)_io__Buffered_name_get, (setter)_io__Buffered_name_set, _io__Buffered_name_DOCSTR}, +#else +# define _IO__BUFFERED_NAME_GETSETDEF {"name", (getter)_io__Buffered_name_get, NULL, _io__Buffered_name_DOCSTR}, +#endif + +static PyObject * +_io__Buffered_name_get_impl(buffered *self); + +static PyObject * +_io__Buffered_name_get(buffered *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_name_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io__Buffered_mode_HAS_DOCSTR) +# define _io__Buffered_mode_DOCSTR _io__Buffered_mode__doc__ +#else +# define _io__Buffered_mode_DOCSTR NULL +#endif +#if defined(_IO__BUFFERED_MODE_GETSETDEF) +# undef _IO__BUFFERED_MODE_GETSETDEF +# define _IO__BUFFERED_MODE_GETSETDEF {"mode", (getter)_io__Buffered_mode_get, (setter)_io__Buffered_mode_set, _io__Buffered_mode_DOCSTR}, +#else +# define _IO__BUFFERED_MODE_GETSETDEF {"mode", (getter)_io__Buffered_mode_get, NULL, _io__Buffered_mode_DOCSTR}, +#endif + +static PyObject * +_io__Buffered_mode_get_impl(buffered *self); + +static PyObject * +_io__Buffered_mode_get(buffered *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io__Buffered_mode_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_io__Buffered_fileno__doc__, "fileno($self, /)\n" "--\n" @@ -1164,4 +1245,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=e8ad39a45531d7f2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=442b05b9a117df6c input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index 8e5c687dc6a55f..6bdb2181985f7d 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -474,4 +474,85 @@ _io_StringIO___setstate__(stringio *self, PyObject *state) return return_value; } -/*[clinic end generated code: output=5c8d67f4408a1e6e input=a9049054013a1b77]*/ + +#if defined(_io_StringIO_closed_HAS_DOCSTR) +# define _io_StringIO_closed_DOCSTR _io_StringIO_closed__doc__ +#else +# define _io_StringIO_closed_DOCSTR NULL +#endif +#if defined(_IO_STRINGIO_CLOSED_GETSETDEF) +# undef _IO_STRINGIO_CLOSED_GETSETDEF +# define _IO_STRINGIO_CLOSED_GETSETDEF {"closed", (getter)_io_StringIO_closed_get, (setter)_io_StringIO_closed_set, _io_StringIO_closed_DOCSTR}, +#else +# define _IO_STRINGIO_CLOSED_GETSETDEF {"closed", (getter)_io_StringIO_closed_get, NULL, _io_StringIO_closed_DOCSTR}, +#endif + +static PyObject * +_io_StringIO_closed_get_impl(stringio *self); + +static PyObject * +_io_StringIO_closed_get(stringio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_closed_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_StringIO_line_buffering_HAS_DOCSTR) +# define _io_StringIO_line_buffering_DOCSTR _io_StringIO_line_buffering__doc__ +#else +# define _io_StringIO_line_buffering_DOCSTR NULL +#endif +#if defined(_IO_STRINGIO_LINE_BUFFERING_GETSETDEF) +# undef _IO_STRINGIO_LINE_BUFFERING_GETSETDEF +# define _IO_STRINGIO_LINE_BUFFERING_GETSETDEF {"line_buffering", (getter)_io_StringIO_line_buffering_get, (setter)_io_StringIO_line_buffering_set, _io_StringIO_line_buffering_DOCSTR}, +#else +# define _IO_STRINGIO_LINE_BUFFERING_GETSETDEF {"line_buffering", (getter)_io_StringIO_line_buffering_get, NULL, _io_StringIO_line_buffering_DOCSTR}, +#endif + +static PyObject * +_io_StringIO_line_buffering_get_impl(stringio *self); + +static PyObject * +_io_StringIO_line_buffering_get(stringio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_line_buffering_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_StringIO_newlines_HAS_DOCSTR) +# define _io_StringIO_newlines_DOCSTR _io_StringIO_newlines__doc__ +#else +# define _io_StringIO_newlines_DOCSTR NULL +#endif +#if defined(_IO_STRINGIO_NEWLINES_GETSETDEF) +# undef _IO_STRINGIO_NEWLINES_GETSETDEF +# define _IO_STRINGIO_NEWLINES_GETSETDEF {"newlines", (getter)_io_StringIO_newlines_get, (setter)_io_StringIO_newlines_set, _io_StringIO_newlines_DOCSTR}, +#else +# define _IO_STRINGIO_NEWLINES_GETSETDEF {"newlines", (getter)_io_StringIO_newlines_get, NULL, _io_StringIO_newlines_DOCSTR}, +#endif + +static PyObject * +_io_StringIO_newlines_get_impl(stringio *self); + +static PyObject * +_io_StringIO_newlines_get(stringio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_StringIO_newlines_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=9ffea20cd32d4cd8 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 675e0ed2eab75e..23b3cc8d71e098 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -201,6 +201,89 @@ _io__TextIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, return return_value; } +PyDoc_STRVAR(_io__TextIOBase_encoding__doc__, +"Encoding of the text stream.\n" +"\n" +"Subclasses should override."); +#define _io__TextIOBase_encoding_HAS_DOCSTR + +#if defined(_io__TextIOBase_encoding_HAS_DOCSTR) +# define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ +#else +# define _io__TextIOBase_encoding_DOCSTR NULL +#endif +#if defined(_IO__TEXTIOBASE_ENCODING_GETSETDEF) +# undef _IO__TEXTIOBASE_ENCODING_GETSETDEF +# define _IO__TEXTIOBASE_ENCODING_GETSETDEF {"encoding", (getter)_io__TextIOBase_encoding_get, (setter)_io__TextIOBase_encoding_set, _io__TextIOBase_encoding_DOCSTR}, +#else +# define _IO__TEXTIOBASE_ENCODING_GETSETDEF {"encoding", (getter)_io__TextIOBase_encoding_get, NULL, _io__TextIOBase_encoding_DOCSTR}, +#endif + +static PyObject * +_io__TextIOBase_encoding_get_impl(PyObject *self); + +static PyObject * +_io__TextIOBase_encoding_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io__TextIOBase_encoding_get_impl(self); +} + +PyDoc_STRVAR(_io__TextIOBase_newlines__doc__, +"Line endings translated so far.\n" +"\n" +"Only line endings translated during reading are considered.\n" +"\n" +"Subclasses should override."); +#define _io__TextIOBase_newlines_HAS_DOCSTR + +#if defined(_io__TextIOBase_newlines_HAS_DOCSTR) +# define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ +#else +# define _io__TextIOBase_newlines_DOCSTR NULL +#endif +#if defined(_IO__TEXTIOBASE_NEWLINES_GETSETDEF) +# undef _IO__TEXTIOBASE_NEWLINES_GETSETDEF +# define _IO__TEXTIOBASE_NEWLINES_GETSETDEF {"newlines", (getter)_io__TextIOBase_newlines_get, (setter)_io__TextIOBase_newlines_set, _io__TextIOBase_newlines_DOCSTR}, +#else +# define _IO__TEXTIOBASE_NEWLINES_GETSETDEF {"newlines", (getter)_io__TextIOBase_newlines_get, NULL, _io__TextIOBase_newlines_DOCSTR}, +#endif + +static PyObject * +_io__TextIOBase_newlines_get_impl(PyObject *self); + +static PyObject * +_io__TextIOBase_newlines_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io__TextIOBase_newlines_get_impl(self); +} + +PyDoc_STRVAR(_io__TextIOBase_errors__doc__, +"The error setting of the decoder or encoder.\n" +"\n" +"Subclasses should override."); +#define _io__TextIOBase_errors_HAS_DOCSTR + +#if defined(_io__TextIOBase_errors_HAS_DOCSTR) +# define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ +#else +# define _io__TextIOBase_errors_DOCSTR NULL +#endif +#if defined(_IO__TEXTIOBASE_ERRORS_GETSETDEF) +# undef _IO__TEXTIOBASE_ERRORS_GETSETDEF +# define _IO__TEXTIOBASE_ERRORS_GETSETDEF {"errors", (getter)_io__TextIOBase_errors_get, (setter)_io__TextIOBase_errors_set, _io__TextIOBase_errors_DOCSTR}, +#else +# define _IO__TEXTIOBASE_ERRORS_GETSETDEF {"errors", (getter)_io__TextIOBase_errors_get, NULL, _io__TextIOBase_errors_DOCSTR}, +#endif + +static PyObject * +_io__TextIOBase_errors_get_impl(PyObject *self); + +static PyObject * +_io__TextIOBase_errors_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _io__TextIOBase_errors_get_impl(self); +} + PyDoc_STRVAR(_io_IncrementalNewlineDecoder___init____doc__, "IncrementalNewlineDecoder(decoder, translate, errors=\'strict\')\n" "--\n" @@ -1047,4 +1130,166 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) return return_value; } -/*[clinic end generated code: output=8781a91be6d99e2c input=a9049054013a1b77]*/ + +#if defined(_io_TextIOWrapper_name_HAS_DOCSTR) +# define _io_TextIOWrapper_name_DOCSTR _io_TextIOWrapper_name__doc__ +#else +# define _io_TextIOWrapper_name_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER_NAME_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_NAME_GETSETDEF +# define _IO_TEXTIOWRAPPER_NAME_GETSETDEF {"name", (getter)_io_TextIOWrapper_name_get, (setter)_io_TextIOWrapper_name_set, _io_TextIOWrapper_name_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER_NAME_GETSETDEF {"name", (getter)_io_TextIOWrapper_name_get, NULL, _io_TextIOWrapper_name_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper_name_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_name_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_name_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_TextIOWrapper_closed_HAS_DOCSTR) +# define _io_TextIOWrapper_closed_DOCSTR _io_TextIOWrapper_closed__doc__ +#else +# define _io_TextIOWrapper_closed_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER_CLOSED_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF +# define _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF {"closed", (getter)_io_TextIOWrapper_closed_get, (setter)_io_TextIOWrapper_closed_set, _io_TextIOWrapper_closed_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF {"closed", (getter)_io_TextIOWrapper_closed_get, NULL, _io_TextIOWrapper_closed_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper_closed_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_closed_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_closed_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_TextIOWrapper_newlines_HAS_DOCSTR) +# define _io_TextIOWrapper_newlines_DOCSTR _io_TextIOWrapper_newlines__doc__ +#else +# define _io_TextIOWrapper_newlines_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF +# define _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF {"newlines", (getter)_io_TextIOWrapper_newlines_get, (setter)_io_TextIOWrapper_newlines_set, _io_TextIOWrapper_newlines_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF {"newlines", (getter)_io_TextIOWrapper_newlines_get, NULL, _io_TextIOWrapper_newlines_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper_newlines_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_newlines_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_newlines_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_TextIOWrapper_errors_HAS_DOCSTR) +# define _io_TextIOWrapper_errors_DOCSTR _io_TextIOWrapper_errors__doc__ +#else +# define _io_TextIOWrapper_errors_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER_ERRORS_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF +# define _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF {"errors", (getter)_io_TextIOWrapper_errors_get, (setter)_io_TextIOWrapper_errors_set, _io_TextIOWrapper_errors_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF {"errors", (getter)_io_TextIOWrapper_errors_get, NULL, _io_TextIOWrapper_errors_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper_errors_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_errors_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_errors_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_io_TextIOWrapper__CHUNK_SIZE_HAS_DOCSTR) +# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ +#else +# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) +# undef _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, _io_TextIOWrapper__CHUNK_SIZE_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, NULL, _io_TextIOWrapper__CHUNK_SIZE_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper__CHUNK_SIZE_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper__CHUNK_SIZE_get(textio *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper__CHUNK_SIZE_get_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_HAS_DOCSTR) +# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ +#else +# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) +# undef _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", (getter)_io_TextIOWrapper__CHUNK_SIZE_get, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, _io_TextIOWrapper__CHUNK_SIZE_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {"_CHUNK_SIZE", NULL, (setter)_io_TextIOWrapper__CHUNK_SIZE_set, NULL}, +#endif + +static int +_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value); + +static int +_io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper__CHUNK_SIZE_set_impl(self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=d01aa598647c1385 input=a9049054013a1b77]*/ diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 0aa5e34cd7c8b2..06bc2679e8e227 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -970,44 +970,44 @@ _io_StringIO___setstate___impl(stringio *self, PyObject *state) Py_RETURN_NONE; } +/*[clinic input] +@critical_section +@getter +_io.StringIO.closed +[clinic start generated code]*/ static PyObject * -stringio_closed_impl(stringio *self, void *context) +_io_StringIO_closed_get_impl(stringio *self) +/*[clinic end generated code: output=531ddca7954331d6 input=178d2ef24395fd49]*/ { CHECK_INITIALIZED(self); return PyBool_FromLong(self->closed); } -static PyObject * -stringio_closed(stringio *self, void *context) -{ - PyObject *result; - Py_BEGIN_CRITICAL_SECTION(self); - result = stringio_closed_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@getter +_io.StringIO.line_buffering +[clinic start generated code]*/ static PyObject * -stringio_line_buffering_impl(stringio *self, void *context) +_io_StringIO_line_buffering_get_impl(stringio *self) +/*[clinic end generated code: output=360710e0112966ae input=6a7634e7f890745e]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); Py_RETURN_FALSE; } -static PyObject * -stringio_line_buffering(stringio *self, void *context) -{ - PyObject *result; - Py_BEGIN_CRITICAL_SECTION(self); - result = stringio_line_buffering_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@getter +_io.StringIO.newlines +[clinic start generated code]*/ static PyObject * -stringio_newlines_impl(stringio *self, void *context) +_io_StringIO_newlines_get_impl(stringio *self) +/*[clinic end generated code: output=35d7c0b66d7e0160 input=092a14586718244b]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -1017,16 +1017,6 @@ stringio_newlines_impl(stringio *self, void *context) return PyObject_GetAttr(self->decoder, &_Py_ID(newlines)); } -static PyObject * -stringio_newlines(stringio *self, void *context) -{ - PyObject *result; - Py_BEGIN_CRITICAL_SECTION(self); - result = stringio_newlines_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} - static struct PyMethodDef stringio_methods[] = { _IO_STRINGIO_CLOSE_METHODDEF _IO_STRINGIO_GETVALUE_METHODDEF @@ -1047,15 +1037,15 @@ static struct PyMethodDef stringio_methods[] = { }; static PyGetSetDef stringio_getset[] = { - {"closed", (getter)stringio_closed, NULL, NULL}, - {"newlines", (getter)stringio_newlines, NULL, NULL}, + _IO_STRINGIO_CLOSED_GETSETDEF + _IO_STRINGIO_NEWLINES_GETSETDEF /* (following comments straight off of the original Python wrapper:) XXX Cruft to support the TextIOWrapper API. This would only be meaningful if StringIO supported the buffer attribute. Hopefully, a better solution, than adding these pseudo-attributes, will be found. */ - {"line_buffering", (getter)stringio_line_buffering, NULL, NULL}, + _IO_STRINGIO_LINE_BUFFERING_GETSETDEF {NULL} }; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 545f467b7f0257..4507930c14bb50 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -131,40 +131,52 @@ _io__TextIOBase_write_impl(PyObject *self, PyTypeObject *cls, return _unsupported(state, "write"); } -PyDoc_STRVAR(textiobase_encoding_doc, - "Encoding of the text stream.\n" - "\n" - "Subclasses should override.\n" - ); +/*[clinic input] +@getter +_io._TextIOBase.encoding + +Encoding of the text stream. + +Subclasses should override. +[clinic start generated code]*/ static PyObject * -textiobase_encoding_get(PyObject *self, void *context) +_io__TextIOBase_encoding_get_impl(PyObject *self) +/*[clinic end generated code: output=e0f5d8f548b92432 input=4736d7621dd38f43]*/ { Py_RETURN_NONE; } -PyDoc_STRVAR(textiobase_newlines_doc, - "Line endings translated so far.\n" - "\n" - "Only line endings translated during reading are considered.\n" - "\n" - "Subclasses should override.\n" - ); +/*[clinic input] +@getter +_io._TextIOBase.newlines + +Line endings translated so far. + +Only line endings translated during reading are considered. + +Subclasses should override. +[clinic start generated code]*/ static PyObject * -textiobase_newlines_get(PyObject *self, void *context) +_io__TextIOBase_newlines_get_impl(PyObject *self) +/*[clinic end generated code: output=46ec147fb9f00c2a input=a5b196d076af1164]*/ { Py_RETURN_NONE; } -PyDoc_STRVAR(textiobase_errors_doc, - "The error setting of the decoder or encoder.\n" - "\n" - "Subclasses should override.\n" - ); +/*[clinic input] +@getter +_io._TextIOBase.errors + +The error setting of the decoder or encoder. + +Subclasses should override. +[clinic start generated code]*/ static PyObject * -textiobase_errors_get(PyObject *self, void *context) +_io__TextIOBase_errors_get_impl(PyObject *self) +/*[clinic end generated code: output=c6623d6addcd087d input=974aa52d1db93a82]*/ { Py_RETURN_NONE; } @@ -179,9 +191,9 @@ static PyMethodDef textiobase_methods[] = { }; static PyGetSetDef textiobase_getset[] = { - {"encoding", (getter)textiobase_encoding_get, NULL, textiobase_encoding_doc}, - {"newlines", (getter)textiobase_newlines_get, NULL, textiobase_newlines_doc}, - {"errors", (getter)textiobase_errors_get, NULL, textiobase_errors_doc}, + _IO__TEXTIOBASE_ENCODING_GETSETDEF + _IO__TEXTIOBASE_NEWLINES_GETSETDEF + _IO__TEXTIOBASE_ERRORS_GETSETDEF {NULL} }; @@ -1475,7 +1487,7 @@ textiowrapper_traverse(textio *self, visitproc visit, void *arg) } static PyObject * -textiowrapper_closed_get(textio *self, void *context); +_io_TextIOWrapper_closed_get_impl(textio *self); /* This macro takes some shortcuts to make the common case faster. */ #define CHECK_CLOSED(self) \ @@ -1486,7 +1498,7 @@ textiowrapper_closed_get(textio *self, void *context); if (self->raw != NULL) \ r = _PyFileIO_closed(self->raw); \ else { \ - _res = textiowrapper_closed_get(self, NULL); \ + _res = _io_TextIOWrapper_closed_get_impl(self); \ if (_res == NULL) \ return NULL; \ r = PyObject_IsTrue(_res); \ @@ -3090,7 +3102,7 @@ _io_TextIOWrapper_close_impl(textio *self) int r; CHECK_ATTACHED(self); - res = textiowrapper_closed_get(self, NULL); + res = _io_TextIOWrapper_closed_get_impl(self); if (res == NULL) return NULL; r = PyObject_IsTrue(res); @@ -3164,42 +3176,43 @@ textiowrapper_iternext(textio *self) return line; } +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.name +[clinic start generated code]*/ + static PyObject * -textiowrapper_name_get_impl(textio *self, void *context) +_io_TextIOWrapper_name_get_impl(textio *self) +/*[clinic end generated code: output=8c2f1d6d8756af40 input=26ecec9b39e30e07]*/ { CHECK_ATTACHED(self); return PyObject_GetAttr(self->buffer, &_Py_ID(name)); } -static PyObject * -textiowrapper_name_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_name_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.closed +[clinic start generated code]*/ static PyObject * -textiowrapper_closed_get_impl(textio *self, void *context) +_io_TextIOWrapper_closed_get_impl(textio *self) +/*[clinic end generated code: output=b49b68f443a85e3c input=7dfcf43f63c7003d]*/ { CHECK_ATTACHED(self); return PyObject_GetAttr(self->buffer, &_Py_ID(closed)); } -static PyObject * -textiowrapper_closed_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_closed_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.newlines +[clinic start generated code]*/ static PyObject * -textiowrapper_newlines_get_impl(textio *self, void *context) +_io_TextIOWrapper_newlines_get_impl(textio *self) +/*[clinic end generated code: output=53aa03ac35573180 input=610df647e514b3e8]*/ { PyObject *res; CHECK_ATTACHED(self); @@ -3211,60 +3224,51 @@ textiowrapper_newlines_get_impl(textio *self, void *context) return res; } -static PyObject * -textiowrapper_newlines_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_newlines_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.errors +[clinic start generated code]*/ static PyObject * -textiowrapper_errors_get_impl(textio *self, void *context) +_io_TextIOWrapper_errors_get_impl(textio *self) +/*[clinic end generated code: output=dca3a3ef21b09484 input=b45f983e6d43c4d8]*/ { CHECK_INITIALIZED(self); return Py_NewRef(self->errors); } -static PyObject * -textiowrapper_errors_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_errors_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper._CHUNK_SIZE +[clinic start generated code]*/ static PyObject * -textiowrapper_chunk_size_get_impl(textio *self, void *context) +_io_TextIOWrapper__CHUNK_SIZE_get_impl(textio *self) +/*[clinic end generated code: output=039925cd2df375bc input=e9715b0e06ff0fa6]*/ { CHECK_ATTACHED(self); return PyLong_FromSsize_t(self->chunk_size); } -static PyObject * -textiowrapper_chunk_size_get(textio *self, void *context) -{ - PyObject *result = NULL; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_chunk_size_get_impl(self, context); - Py_END_CRITICAL_SECTION(); - return result; -} +/*[clinic input] +@critical_section +@setter +_io.TextIOWrapper._CHUNK_SIZE +[clinic start generated code]*/ static int -textiowrapper_chunk_size_set_impl(textio *self, PyObject *arg, void *context) +_io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value) +/*[clinic end generated code: output=edb86d2db660a5ab input=32fc99861db02a0a]*/ { Py_ssize_t n; CHECK_ATTACHED_INT(self); - if (arg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - n = PyNumber_AsSsize_t(arg, PyExc_ValueError); + n = PyNumber_AsSsize_t(value, PyExc_ValueError); if (n == -1 && PyErr_Occurred()) return -1; if (n <= 0) { @@ -3276,16 +3280,6 @@ textiowrapper_chunk_size_set_impl(textio *self, PyObject *arg, void *context) return 0; } -static int -textiowrapper_chunk_size_set(textio *self, PyObject *arg, void *context) -{ - int result = 0; - Py_BEGIN_CRITICAL_SECTION(self); - result = textiowrapper_chunk_size_set_impl(self, arg, context); - Py_END_CRITICAL_SECTION(); - return result; -} - static PyMethodDef incrementalnewlinedecoder_methods[] = { _IO_INCREMENTALNEWLINEDECODER_DECODE_METHODDEF _IO_INCREMENTALNEWLINEDECODER_GETSTATE_METHODDEF @@ -3355,14 +3349,13 @@ static PyMemberDef textiowrapper_members[] = { }; static PyGetSetDef textiowrapper_getset[] = { - {"name", (getter)textiowrapper_name_get, NULL, NULL}, - {"closed", (getter)textiowrapper_closed_get, NULL, NULL}, + _IO_TEXTIOWRAPPER_NAME_GETSETDEF + _IO_TEXTIOWRAPPER_CLOSED_GETSETDEF /* {"mode", (getter)TextIOWrapper_mode_get, NULL, NULL}, */ - {"newlines", (getter)textiowrapper_newlines_get, NULL, NULL}, - {"errors", (getter)textiowrapper_errors_get, NULL, NULL}, - {"_CHUNK_SIZE", (getter)textiowrapper_chunk_size_get, - (setter)textiowrapper_chunk_size_set, NULL}, + _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF + _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF + _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF {NULL} }; diff --git a/Modules/_json.c b/Modules/_json.c index 0b1bfe34ad9304..24b292ce70e5eb 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -662,6 +662,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss PyObject *key = NULL; int has_pairs_hook = (s->object_pairs_hook != Py_None); Py_ssize_t next_idx; + Py_ssize_t comma_idx; str = PyUnicode_DATA(pystr); kind = PyUnicode_KIND(pystr); @@ -741,10 +742,16 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss raise_errmsg("Expecting ',' delimiter", pystr, idx); goto bail; } + comma_idx = idx; idx++; /* skip whitespace after , delimiter */ while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; + + if (idx <= end_idx && PyUnicode_READ(kind, str, idx) == '}') { + raise_errmsg("Illegal trailing comma before end of object", pystr, comma_idx); + goto bail; + } } } @@ -785,6 +792,7 @@ _parse_array_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssi PyObject *val = NULL; PyObject *rval; Py_ssize_t next_idx; + Py_ssize_t comma_idx; rval = PyList_New(0); if (rval == NULL) @@ -822,10 +830,16 @@ _parse_array_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssi raise_errmsg("Expecting ',' delimiter", pystr, idx); goto bail; } + comma_idx = idx; idx++; /* skip whitespace after , */ while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; + + if (idx <= end_idx && PyUnicode_READ(kind, str, idx) == ']') { + raise_errmsg("Illegal trailing comma before end of array", pystr, comma_idx); + goto bail; + } } } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index a3cf34699ba509..227e5378e42285 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4707,6 +4707,14 @@ Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) Py_VISIT(self->fast_memo); Py_VISIT(self->reducer_override); Py_VISIT(self->buffer_callback); + PyMemoTable *memo = self->memo; + if (memo && memo->mt_table) { + Py_ssize_t i = memo->mt_allocated; + while (--i >= 0) { + Py_VISIT(memo->mt_table[i].me_key); + } + } + return 0; } @@ -7175,6 +7183,13 @@ Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg) Py_VISIT(self->stack); Py_VISIT(self->pers_func); Py_VISIT(self->buffers); + PyObject **memo = self->memo; + if (memo) { + Py_ssize_t i = self->memo_size; + while (--i >= 0) { + Py_VISIT(memo[i]); + } + } return 0; } diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 2898eedc3e3a8f..d0dd8f064e0395 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -767,8 +767,10 @@ child_exec(char *const exec_array[], #endif #ifdef HAVE_SETGROUPS - if (extra_group_size > 0) + if (extra_group_size >= 0) { + assert((extra_group_size == 0) == (extra_groups == NULL)); POSIX_CALL(setgroups(extra_group_size, extra_groups)); + } #endif /* HAVE_SETGROUPS */ #ifdef HAVE_SETREGID @@ -1022,7 +1024,6 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, pid_t pid = -1; int need_to_reenable_gc = 0; char *const *argv = NULL, *const *envp = NULL; - Py_ssize_t extra_group_size = 0; int need_after_fork = 0; int saved_errno = 0; int *c_fds_to_keep = NULL; @@ -1103,6 +1104,13 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, cwd = PyBytes_AsString(cwd_obj2); } + // Special initial value meaning that subprocess API was called with + // extra_groups=None leading to _posixsubprocess.fork_exec(gids=None). + // We use this to differentiate between code desiring a setgroups(0, NULL) + // call vs no call at all. The fast vfork() code path could be used when + // there is no setgroups call. + Py_ssize_t extra_group_size = -2; + if (extra_groups_packed != Py_None) { #ifdef HAVE_SETGROUPS if (!PyList_Check(extra_groups_packed)) { diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 514bec16a347f4..4403e1d132c057 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -175,6 +175,7 @@ genrand_uint32(RandomObject *self) */ /*[clinic input] +@critical_section _random.Random.random self: self(type="RandomObject *") @@ -184,7 +185,7 @@ random() -> x in the interval [0, 1). static PyObject * _random_Random_random_impl(RandomObject *self) -/*[clinic end generated code: output=117ff99ee53d755c input=afb2a59cbbb00349]*/ +/*[clinic end generated code: output=117ff99ee53d755c input=26492e52d26e8b7b]*/ { uint32_t a=genrand_uint32(self)>>5, b=genrand_uint32(self)>>6; return PyFloat_FromDouble((a*67108864.0+b)*(1.0/9007199254740992.0)); @@ -368,6 +369,7 @@ random_seed(RandomObject *self, PyObject *arg) } /*[clinic input] +@critical_section _random.Random.seed self: self(type="RandomObject *") @@ -382,7 +384,7 @@ of the current time and the process identifier. static PyObject * _random_Random_seed_impl(RandomObject *self, PyObject *n) -/*[clinic end generated code: output=0fad1e16ba883681 input=78d6ef0d52532a54]*/ +/*[clinic end generated code: output=0fad1e16ba883681 input=46d01d2ba938c7b1]*/ { if (random_seed(self, n) < 0) { return NULL; @@ -391,6 +393,7 @@ _random_Random_seed_impl(RandomObject *self, PyObject *n) } /*[clinic input] +@critical_section _random.Random.getstate self: self(type="RandomObject *") @@ -400,7 +403,7 @@ getstate() -> tuple containing the current state. static PyObject * _random_Random_getstate_impl(RandomObject *self) -/*[clinic end generated code: output=bf6cef0c092c7180 input=b937a487928c0e89]*/ +/*[clinic end generated code: output=bf6cef0c092c7180 input=b6621f31eb639694]*/ { PyObject *state; PyObject *element; @@ -428,6 +431,7 @@ _random_Random_getstate_impl(RandomObject *self) /*[clinic input] +@critical_section _random.Random.setstate self: self(type="RandomObject *") @@ -438,8 +442,8 @@ setstate(state) -> None. Restores generator state. [clinic start generated code]*/ static PyObject * -_random_Random_setstate(RandomObject *self, PyObject *state) -/*[clinic end generated code: output=fd1c3cd0037b6681 input=b3b4efbb1bc66af8]*/ +_random_Random_setstate_impl(RandomObject *self, PyObject *state) +/*[clinic end generated code: output=babfc2c2eac6b027 input=358e898ec07469b7]*/ { int i; unsigned long element; @@ -479,7 +483,7 @@ _random_Random_setstate(RandomObject *self, PyObject *state) } /*[clinic input] - +@critical_section _random.Random.getrandbits self: self(type="RandomObject *") @@ -491,7 +495,7 @@ getrandbits(k) -> x. Generates an int with k random bits. static PyObject * _random_Random_getrandbits_impl(RandomObject *self, int k) -/*[clinic end generated code: output=b402f82a2158887f input=8c0e6396dd176fc0]*/ +/*[clinic end generated code: output=b402f82a2158887f input=87603cd60f79f730]*/ { int i, words; uint32_t r; diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 7920d2c2b8739d..fe82e918677f9a 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -3,9 +3,7 @@ * using the SystemConfiguration framework. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 707e7ad9543acb..04c9f7daadf573 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -301,8 +301,10 @@ typedef struct { BIO *keylog_bio; /* Cached module state, also used in SSLSocket and SSLSession code. */ _sslmodulestate *state; +#ifndef OPENSSL_NO_PSK PyObject *psk_client_callback; PyObject *psk_server_callback; +#endif } PySSLContext; typedef struct { @@ -891,10 +893,8 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, * only in combination with SSL_VERIFY_PEER flag. */ int mode = SSL_get_verify_mode(self->ssl); if (mode & SSL_VERIFY_PEER) { - int (*verify_cb)(int, X509_STORE_CTX *) = NULL; - verify_cb = SSL_get_verify_callback(self->ssl); mode |= SSL_VERIFY_POST_HANDSHAKE; - SSL_set_verify(self->ssl, mode, verify_cb); + SSL_set_verify(self->ssl, mode, NULL); } } else { /* client socket */ @@ -2995,7 +2995,6 @@ static int _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) { int mode; - int (*verify_cb)(int, X509_STORE_CTX *) = NULL; switch(n) { case PY_SSL_CERT_NONE: @@ -3016,9 +3015,7 @@ _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) /* bpo-37428: newPySSLSocket() sets SSL_VERIFY_POST_HANDSHAKE flag for * server sockets and SSL_set_post_handshake_auth() for client. */ - /* keep current verify cb */ - verify_cb = SSL_CTX_get_verify_callback(self->ctx); - SSL_CTX_set_verify(self->ctx, mode, verify_cb); + SSL_CTX_set_verify(self->ctx, mode, NULL); return 0; } @@ -3125,8 +3122,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) self->alpn_protocols = NULL; self->set_sni_cb = NULL; self->state = get_ssl_state(module); +#ifndef OPENSSL_NO_PSK self->psk_client_callback = NULL; self->psk_server_callback = NULL; +#endif /* Don't check host name by default */ if (proto_version == PY_SSL_VERSION_TLS_CLIENT) { @@ -3239,8 +3238,10 @@ context_clear(PySSLContext *self) Py_CLEAR(self->set_sni_cb); Py_CLEAR(self->msg_cb); Py_CLEAR(self->keylog_filename); +#ifndef OPENSSL_NO_PSK Py_CLEAR(self->psk_client_callback); Py_CLEAR(self->psk_server_callback); +#endif if (self->keylog_bio != NULL) { PySSL_BEGIN_ALLOW_THREADS BIO_free_all(self->keylog_bio); @@ -4668,6 +4669,7 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) return NULL; } +#ifndef OPENSSL_NO_PSK static unsigned int psk_client_callback(SSL *s, const char *hint, char *identity, @@ -4735,6 +4737,7 @@ static unsigned int psk_client_callback(SSL *s, PyGILState_Release(gstate); return 0; } +#endif /*[clinic input] _ssl._SSLContext.set_psk_client_callback @@ -4747,6 +4750,7 @@ _ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, PyObject *callback) /*[clinic end generated code: output=0aba86f6ed75119e input=7627bae0e5ee7635]*/ { +#ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_SERVER) { _setSSLError(get_state_ctx(self), "Cannot add PSK client callback to a " @@ -4774,8 +4778,14 @@ _ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, SSL_CTX_set_psk_client_callback(self->ctx, ssl_callback); Py_RETURN_NONE; +#else + PyErr_SetString(PyExc_NotImplementedError, + "TLS-PSK is not supported by your OpenSSL version."); + return NULL; +#endif } +#ifndef OPENSSL_NO_PSK static unsigned int psk_server_callback(SSL *s, const char *identity, unsigned char *psk, @@ -4835,6 +4845,7 @@ static unsigned int psk_server_callback(SSL *s, PyGILState_Release(gstate); return 0; } +#endif /*[clinic input] _ssl._SSLContext.set_psk_server_callback @@ -4849,6 +4860,7 @@ _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, const char *identity_hint) /*[clinic end generated code: output=1f4d6a4e09a92b03 input=65d4b6022aa85ea3]*/ { +#ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { _setSSLError(get_state_ctx(self), "Cannot add PSK server callback to a " @@ -4882,6 +4894,11 @@ _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, SSL_CTX_set_psk_server_callback(self->ctx, ssl_callback); Py_RETURN_NONE; +#else + PyErr_SetString(PyExc_NotImplementedError, + "TLS-PSK is not supported by your OpenSSL version."); + return NULL; +#endif } @@ -6243,6 +6260,12 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_TLSv1_3", 0); #endif +#ifdef OPENSSL_NO_PSK + addbool(m, "HAS_PSK", 0); +#else + addbool(m, "HAS_PSK", 1); +#endif + #undef addbool #undef ADD_INT_CONST diff --git a/Modules/_stat.c b/Modules/_stat.c index 1ef1e97f4b7dca..80f8a92668976b 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -11,9 +11,7 @@ * */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.13 for PyModule_Add() on Windows diff --git a/Modules/_struct.c b/Modules/_struct.c index 24a4cb3b6413f1..bd16fa89f18945 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2257,14 +2257,13 @@ cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) return 1; } - s_object = PyDict_GetItemWithError(state->cache, fmt); + if (PyDict_GetItemRef(state->cache, fmt, &s_object) < 0) { + return 0; + } if (s_object != NULL) { - *ptr = (PyStructObject *)Py_NewRef(s_object); + *ptr = (PyStructObject *)s_object; return Py_CLEANUP_SUPPORTED; } - else if (PyErr_Occurred()) { - return 0; - } s_object = PyObject_CallOneArg(state->PyStructType, fmt); if (s_object != NULL) { diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index 4a9144e66f0fcd..a8ba009eb6a54b 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -2,6 +2,34 @@ #include "util.h" +static PyObject * +object_repr(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Repr(arg); +} + +static PyObject * +object_ascii(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_ASCII(arg); +} + +static PyObject * +object_str(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Str(arg); +} + +static PyObject * +object_bytes(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Bytes(arg); +} + static PyObject * object_getattr(PyObject *self, PyObject *args) { @@ -616,6 +644,11 @@ sequence_tuple(PyObject *self, PyObject *obj) static PyMethodDef test_methods[] = { + {"object_repr", object_repr, METH_O}, + {"object_ascii", object_ascii, METH_O}, + {"object_str", object_str, METH_O}, + {"object_bytes", object_bytes, METH_O}, + {"object_getattr", object_getattr, METH_VARARGS}, {"object_getattrstring", object_getattrstring, METH_VARARGS}, {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, diff --git a/Modules/_testcapi/complex.c b/Modules/_testcapi/complex.c index 400f4054c613ee..4a70217eb90d62 100644 --- a/Modules/_testcapi/complex.c +++ b/Modules/_testcapi/complex.c @@ -85,6 +85,58 @@ complex_asccomplex(PyObject *Py_UNUSED(module), PyObject *obj) return PyComplex_FromCComplex(complex); } +static PyObject* +_py_c_neg(PyObject *Py_UNUSED(module), PyObject *num) +{ + Py_complex complex; + + complex = PyComplex_AsCComplex(num); + if (complex.real == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyComplex_FromCComplex(_Py_c_neg(complex)); +} + +#define _PY_C_FUNC2(suffix) \ + static PyObject * \ + _py_c_##suffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + Py_complex num, exp, res; \ + \ + if (!PyArg_ParseTuple(args, "DD", &num, &exp)) { \ + return NULL; \ + } \ + \ + errno = 0; \ + res = _Py_c_##suffix(num, exp); \ + return Py_BuildValue("Di", &res, errno); \ + }; + +_PY_C_FUNC2(sum) +_PY_C_FUNC2(diff) +_PY_C_FUNC2(prod) +_PY_C_FUNC2(quot) +_PY_C_FUNC2(pow) + +static PyObject* +_py_c_abs(PyObject *Py_UNUSED(module), PyObject* obj) +{ + Py_complex complex; + double res; + + NULLABLE(obj); + complex = PyComplex_AsCComplex(obj); + + if (complex.real == -1. && PyErr_Occurred()) { + return NULL; + } + + errno = 0; + res = _Py_c_abs(complex); + return Py_BuildValue("di", res, errno); +} + static PyMethodDef test_methods[] = { {"complex_check", complex_check, METH_O}, @@ -94,6 +146,13 @@ static PyMethodDef test_methods[] = { {"complex_realasdouble", complex_realasdouble, METH_O}, {"complex_imagasdouble", complex_imagasdouble, METH_O}, {"complex_asccomplex", complex_asccomplex, METH_O}, + {"_py_c_sum", _py_c_sum, METH_VARARGS}, + {"_py_c_diff", _py_c_diff, METH_VARARGS}, + {"_py_c_neg", _py_c_neg, METH_O}, + {"_py_c_prod", _py_c_prod, METH_VARARGS}, + {"_py_c_quot", _py_c_quot, METH_VARARGS}, + {"_py_c_pow", _py_c_pow, METH_VARARGS}, + {"_py_c_abs", _py_c_abs, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 88f992915fa8c1..b1796039f0d83a 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -85,17 +85,25 @@ make_timezones_capi(PyObject *self, PyObject *args) { PyObject *offset = PyDelta_FromDSU(0, -18000, 0); PyObject *name = PyUnicode_FromString("EST"); + if (offset == NULL || name == NULL) { + Py_XDECREF(offset); + Py_XDECREF(name); + return NULL; + } PyObject *est_zone_capi = PyDateTimeAPI->TimeZone_FromTimeZone(offset, name); PyObject *est_zone_macro = PyTimeZone_FromOffsetAndName(offset, name); PyObject *est_zone_macro_noname = PyTimeZone_FromOffset(offset); - - Py_DecRef(offset); - Py_DecRef(name); - + Py_DECREF(offset); + Py_DECREF(name); + if (est_zone_capi == NULL || est_zone_macro == NULL || + est_zone_macro_noname == NULL) + { + goto error; + } PyObject *rv = PyTuple_New(3); if (rv == NULL) { - return NULL; + goto error; } PyTuple_SET_ITEM(rv, 0, est_zone_capi); @@ -103,6 +111,11 @@ make_timezones_capi(PyObject *self, PyObject *args) PyTuple_SET_ITEM(rv, 2, est_zone_macro_noname); return rv; +error: + Py_XDECREF(est_zone_capi); + Py_XDECREF(est_zone_macro); + Py_XDECREF(est_zone_macro_noname); + return NULL; } static PyObject * @@ -110,6 +123,11 @@ get_timezones_offset_zero(PyObject *self, PyObject *args) { PyObject *offset = PyDelta_FromDSU(0, 0, 0); PyObject *name = PyUnicode_FromString(""); + if (offset == NULL || name == NULL) { + Py_XDECREF(offset); + Py_XDECREF(name); + return NULL; + } // These two should return the UTC singleton PyObject *utc_singleton_0 = PyTimeZone_FromOffset(offset); @@ -117,16 +135,28 @@ get_timezones_offset_zero(PyObject *self, PyObject *args) // This one will return +00:00 zone, but not the UTC singleton PyObject *non_utc_zone = PyTimeZone_FromOffsetAndName(offset, name); - - Py_DecRef(offset); - Py_DecRef(name); + Py_DECREF(offset); + Py_DECREF(name); + if (utc_singleton_0 == NULL || utc_singleton_1 == NULL || + non_utc_zone == NULL) + { + goto error; + } PyObject *rv = PyTuple_New(3); + if (rv == NULL) { + goto error; + } PyTuple_SET_ITEM(rv, 0, utc_singleton_0); PyTuple_SET_ITEM(rv, 1, utc_singleton_1); PyTuple_SET_ITEM(rv, 2, non_utc_zone); return rv; +error: + Py_XDECREF(utc_singleton_0); + Py_XDECREF(utc_singleton_1); + Py_XDECREF(non_utc_zone); + return NULL; } static PyObject * diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index e4cd15503fd11f..33e8af7d7bbb39 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -71,18 +71,22 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) if (result) { int objects_only = 1; + int count = 0; for (const char *f = sub_format; *f; f++) { - if (Py_ISALNUM(*f) && strchr("OSUY", *f) == NULL) { - objects_only = 0; - break; + if (Py_ISALNUM(*f)) { + if (strchr("OSUY", *f) == NULL) { + objects_only = 0; + break; + } + count++; } } if (objects_only) { - return_value = PyTuple_New(size); + return_value = PyTuple_New(count); if (return_value == NULL) { goto exit; } - for (Py_ssize_t i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < count; i++) { PyObject *arg = *(PyObject **)(buffers + i); if (arg == NULL) { arg = Py_None; diff --git a/Modules/_testcapi/hash.c b/Modules/_testcapi/hash.c index d0b8127020c5c1..aee76787dcddb3 100644 --- a/Modules/_testcapi/hash.c +++ b/Modules/_testcapi/hash.c @@ -44,8 +44,24 @@ hash_getfuncdef(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) return result; } + +static PyObject * +hash_pointer(PyObject *Py_UNUSED(module), PyObject *arg) +{ + void *ptr = PyLong_AsVoidPtr(arg); + if (ptr == NULL && PyErr_Occurred()) { + return NULL; + } + + Py_hash_t hash = Py_HashPointer(ptr); + Py_BUILD_ASSERT(sizeof(long long) >= sizeof(hash)); + return PyLong_FromLongLong(hash); +} + + static PyMethodDef test_methods[] = { {"hash_getfuncdef", hash_getfuncdef, METH_NOARGS}, + {"hash_pointer", hash_pointer, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c index 52286f05f7154c..52bda75736b316 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testcapi/heaptype_relative.c @@ -1,6 +1,4 @@ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x030c0000 // 3.12 diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testcapi/vectorcall_limited.c index 0a650f1b351d2d..d7b8d33b7f7162 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testcapi/vectorcall_limited.c @@ -1,8 +1,6 @@ /* Test Vectorcall in the limited API */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x030c0000 // 3.12 diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 9fdd67093338e4..6762c611fb12a2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -13,6 +13,7 @@ #include "_testcapi/parts.h" #include "frameobject.h" // PyFrame_New() +#include "interpreteridobject.h" // PyInterpreterID_Type #include "marshal.h" // PyMarshal_WriteLongToFile() #include // FLT_MAX @@ -32,15 +33,32 @@ // Forward declarations static struct PyModuleDef _testcapimodule; -static PyObject *TestError; /* set to exception object in init */ +// Module state +typedef struct { + PyObject *error; // _testcapi.error object +} testcapistate_t; -/* Raise TestError with test_name + ": " + msg, and return NULL. */ +static testcapistate_t* +get_testcapi_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (testcapistate_t *)state; +} static PyObject * -raiseTestError(const char* test_name, const char* msg) +get_testerror(PyObject *self) { + testcapistate_t *state = get_testcapi_state((PyObject *)Py_TYPE(self)); + return state->error; +} + +/* Raise _testcapi.error with test_name + ": " + msg, and return NULL. */ + +static PyObject * +raiseTestError(PyObject *self, const char* test_name, const char* msg) { - PyErr_Format(TestError, "%s: %s", test_name, msg); + PyErr_Format(get_testerror(self), "%s: %s", test_name, msg); return NULL; } @@ -51,10 +69,10 @@ raiseTestError(const char* test_name, const char* msg) platforms have these hardcoded. Better safe than sorry. */ static PyObject* -sizeof_error(const char* fatname, const char* typname, +sizeof_error(PyObject *self, const char* fatname, const char* typname, int expected, int got) { - PyErr_Format(TestError, + PyErr_Format(get_testerror(self), "%s #define == %d but sizeof(%s) == %d", fatname, expected, typname, got); return (PyObject*)NULL; @@ -65,7 +83,7 @@ test_config(PyObject *self, PyObject *Py_UNUSED(ignored)) { #define CHECK_SIZEOF(FATNAME, TYPE) \ if (FATNAME != sizeof(TYPE)) \ - return sizeof_error(#FATNAME, #TYPE, FATNAME, sizeof(TYPE)) + return sizeof_error(self, #FATNAME, #TYPE, FATNAME, sizeof(TYPE)) CHECK_SIZEOF(SIZEOF_SHORT, short); CHECK_SIZEOF(SIZEOF_INT, int); @@ -88,7 +106,7 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) #endif #define CHECK_SIZEOF(TYPE, EXPECTED) \ if (EXPECTED != sizeof(TYPE)) { \ - PyErr_Format(TestError, \ + PyErr_Format(get_testerror(self), \ "sizeof(%s) = %u instead of %u", \ #TYPE, sizeof(TYPE), EXPECTED); \ return (PyObject*)NULL; \ @@ -96,7 +114,7 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) #define IS_SIGNED(TYPE) (((TYPE)-1) < (TYPE)0) #define CHECK_SIGNNESS(TYPE, SIGNED) \ if (IS_SIGNED(TYPE) != SIGNED) { \ - PyErr_Format(TestError, \ + PyErr_Format(get_testerror(self), \ "%s signness is, instead of %i", \ #TYPE, IS_SIGNED(TYPE), SIGNED); \ return (PyObject*)NULL; \ @@ -169,7 +187,7 @@ test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) for (i = 0; i < NLIST; ++i) { PyObject* anint = PyList_GET_ITEM(list, i); if (PyLong_AS_LONG(anint) != NLIST-1-i) { - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_list_api: reverse screwed up"); Py_DECREF(list); return (PyObject*)NULL; @@ -182,7 +200,7 @@ test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) } static int -test_dict_inner(int count) +test_dict_inner(PyObject *self, int count) { Py_ssize_t pos = 0, iterations = 0; int i; @@ -195,11 +213,11 @@ test_dict_inner(int count) for (i = 0; i < count; i++) { v = PyLong_FromLong(i); if (v == NULL) { - return -1; + goto error; } if (PyDict_SetItem(dict, v, v) < 0) { Py_DECREF(v); - return -1; + goto error; } Py_DECREF(v); } @@ -213,11 +231,12 @@ test_dict_inner(int count) assert(v != UNINITIALIZED_PTR); i = PyLong_AS_LONG(v) + 1; o = PyLong_FromLong(i); - if (o == NULL) - return -1; + if (o == NULL) { + goto error; + } if (PyDict_SetItem(dict, k, o) < 0) { Py_DECREF(o); - return -1; + goto error; } Py_DECREF(o); k = v = UNINITIALIZED_PTR; @@ -229,12 +248,15 @@ test_dict_inner(int count) if (iterations != count) { PyErr_SetString( - TestError, + get_testerror(self), "test_dict_iteration: dict iteration went wrong "); return -1; } else { return 0; } +error: + Py_DECREF(dict); + return -1; } @@ -245,7 +267,7 @@ test_dict_iteration(PyObject* self, PyObject *Py_UNUSED(ignored)) int i; for (i = 0; i < 200; i++) { - if (test_dict_inner(i) < 0) { + if (test_dict_inner(self, i) < 0) { return NULL; } } @@ -329,14 +351,14 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if (obj == NULL) { PyErr_Clear(); PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: failed to create object"); return NULL; } if (type->tp_dict != NULL) { PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: type initialised too soon"); Py_DECREF(obj); return NULL; @@ -346,7 +368,7 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if ((hash == -1) && PyErr_Occurred()) { PyErr_Clear(); PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: could not hash object"); Py_DECREF(obj); return NULL; @@ -354,7 +376,7 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if (type->tp_dict == NULL) { PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: type not initialised by hash()"); Py_DECREF(obj); return NULL; @@ -362,7 +384,7 @@ test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) if (type->tp_hash != PyType_Type.tp_hash) { PyErr_SetString( - TestError, + get_testerror(self), "test_lazy_hash_inheritance: unexpected hash function"); Py_DECREF(obj); return NULL; @@ -422,7 +444,7 @@ py_buildvalue_ints(PyObject *self, PyObject *args) } static int -test_buildvalue_N_error(const char *fmt) +test_buildvalue_N_error(PyObject *self, const char *fmt) { PyObject *arg, *res; @@ -438,7 +460,7 @@ test_buildvalue_N_error(const char *fmt) } Py_DECREF(res); if (Py_REFCNT(arg) != 1) { - PyErr_Format(TestError, "test_buildvalue_N: " + PyErr_Format(get_testerror(self), "test_buildvalue_N: " "arg was not decrefed in successful " "Py_BuildValue(\"%s\")", fmt); return -1; @@ -447,13 +469,13 @@ test_buildvalue_N_error(const char *fmt) Py_INCREF(arg); res = Py_BuildValue(fmt, raise_error, NULL, arg); if (res != NULL || !PyErr_Occurred()) { - PyErr_Format(TestError, "test_buildvalue_N: " + PyErr_Format(get_testerror(self), "test_buildvalue_N: " "Py_BuildValue(\"%s\") didn't complain", fmt); return -1; } PyErr_Clear(); if (Py_REFCNT(arg) != 1) { - PyErr_Format(TestError, "test_buildvalue_N: " + PyErr_Format(get_testerror(self), "test_buildvalue_N: " "arg was not decrefed in failed " "Py_BuildValue(\"%s\")", fmt); return -1; @@ -477,25 +499,25 @@ test_buildvalue_N(PyObject *self, PyObject *Py_UNUSED(ignored)) return NULL; } if (res != arg) { - return raiseTestError("test_buildvalue_N", + return raiseTestError(self, "test_buildvalue_N", "Py_BuildValue(\"N\") returned wrong result"); } if (Py_REFCNT(arg) != 2) { - return raiseTestError("test_buildvalue_N", + return raiseTestError(self, "test_buildvalue_N", "arg was not decrefed in Py_BuildValue(\"N\")"); } Py_DECREF(res); Py_DECREF(arg); - if (test_buildvalue_N_error("O&N") < 0) + if (test_buildvalue_N_error(self, "O&N") < 0) return NULL; - if (test_buildvalue_N_error("(O&N)") < 0) + if (test_buildvalue_N_error(self, "(O&N)") < 0) return NULL; - if (test_buildvalue_N_error("[O&N]") < 0) + if (test_buildvalue_N_error(self, "[O&N]") < 0) return NULL; - if (test_buildvalue_N_error("{O&N}") < 0) + if (test_buildvalue_N_error(self, "{O&N}") < 0) return NULL; - if (test_buildvalue_N_error("{()O&(())N}") < 0) + if (test_buildvalue_N_error(self, "{()O&(())N}") < 0) return NULL; Py_RETURN_NONE; @@ -905,7 +927,7 @@ test_string_to_double(PyObject *self, PyObject *Py_UNUSED(ignored)) { Py_RETURN_NONE; fail: - return raiseTestError("test_string_to_double", msg); + return raiseTestError(self, "test_string_to_double", msg); #undef CHECK_STRING #undef CHECK_INVALID } @@ -1056,7 +1078,7 @@ test_capsule(PyObject *self, PyObject *Py_UNUSED(ignored)) exit: if (error) { - return raiseTestError("test_capsule", error); + return raiseTestError(self, "test_capsule", error); } Py_RETURN_NONE; #undef FAIL @@ -1267,7 +1289,7 @@ test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) ptr = view.buf; for (i = 0; i < 5; i++) { if (ptr[2*i] != i) { - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_from_contiguous: incorrect result"); return NULL; } @@ -1280,7 +1302,7 @@ test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) ptr = view.buf; for (i = 0; i < 5; i++) { if (*(ptr-2*i) != i) { - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_from_contiguous: incorrect result"); return NULL; } @@ -1333,7 +1355,7 @@ test_pep3118_obsolete_write_locks(PyObject* self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; error: - PyErr_SetString(TestError, + PyErr_SetString(get_testerror(self), "test_pep3118_obsolete_write_locks: failure"); return NULL; } @@ -1451,6 +1473,36 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } +static PyObject * +get_interpreterid_type(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return Py_NewRef(&PyInterpreterID_Type); +} + +static PyObject * +link_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 1); + Py_RETURN_NONE; +} + +static PyObject * +unlink_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 0); + Py_RETURN_NONE; +} + static PyMethodDef ml; static PyObject * @@ -1525,7 +1577,9 @@ test_structseq_newtype_doesnt_leak(PyObject *Py_UNUSED(self), descr.n_in_sequence = 1; PyTypeObject* structseq_type = PyStructSequence_NewType(&descr); - assert(structseq_type != NULL); + if (structseq_type == NULL) { + return NULL; + } assert(PyType_Check(structseq_type)); assert(PyType_FastSubclass(structseq_type, Py_TPFLAGS_TUPLE_SUBCLASS)); Py_DECREF(structseq_type); @@ -1922,7 +1976,7 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) { Py_tss_t tss_key = Py_tss_NEEDS_INIT; if (PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "TSS key not in an uninitialized state at " "creation time"); } @@ -1931,19 +1985,19 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) return NULL; } if (!PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "PyThread_tss_create succeeded, " "but with TSS key in an uninitialized state"); } if (PyThread_tss_create(&tss_key) != 0) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "PyThread_tss_create unsuccessful with " "an already initialized key"); } #define CHECK_TSS_API(expr) \ (void)(expr); \ if (!PyThread_tss_is_created(&tss_key)) { \ - return raiseTestError("test_pythread_tss_key_state", \ + return raiseTestError(self, "test_pythread_tss_key_state", \ "TSS key initialization state was not " \ "preserved after calling " #expr); } CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); @@ -1951,7 +2005,7 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) #undef CHECK_TSS_API PyThread_tss_delete(&tss_key); if (PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "PyThread_tss_delete called, but did not " "set the key state to uninitialized"); } @@ -1962,7 +2016,7 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) return NULL; } if (PyThread_tss_is_created(ptr_key)) { - return raiseTestError("test_pythread_tss_key_state", + return raiseTestError(self, "test_pythread_tss_key_state", "TSS key not in an uninitialized state at " "allocation time"); } @@ -3237,6 +3291,9 @@ static PyMethodDef TestMethods[] = { {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, + {"get_interpreterid_type", get_interpreterid_type, METH_NOARGS}, + {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, + {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, @@ -3791,14 +3848,9 @@ static PyTypeObject ContainerNoGC_type = { static struct PyModuleDef _testcapimodule = { PyModuleDef_HEAD_INIT, - "_testcapi", - NULL, - -1, - TestMethods, - NULL, - NULL, - NULL, - NULL + .m_name = "_testcapi", + .m_size = sizeof(testcapistate_t), + .m_methods = TestMethods, }; /* Per PEP 489, this module will not be converted to multi-phase initialization @@ -3893,9 +3945,10 @@ PyInit__testcapi(void) PyModule_AddIntConstant(m, "the_number_three", 3); PyModule_AddIntMacro(m, Py_C_RECURSION_LIMIT); - TestError = PyErr_NewException("_testcapi.error", NULL, NULL); - Py_INCREF(TestError); - PyModule_AddObject(m, "error", TestError); + testcapistate_t *state = get_testcapi_state(m); + state->error = PyErr_NewException("_testcapi.error", NULL, NULL); + Py_INCREF(state->error); + PyModule_AddObject(m, "error", state->error); if (PyType_Ready(&ContainerNoGC_type) < 0) { return NULL; diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index 61bc84134458da..ef595be0b626db 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -4,9 +4,7 @@ #undef Py_BUILD_CORE_MODULE #undef Py_BUILD_CORE_BUILTIN -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // For now, only limited C API 3.13 is supported diff --git a/Modules/_testimportmultiple.c b/Modules/_testimportmultiple.c index 245e81b2dce7f8..7e6556ad400cde 100644 --- a/Modules/_testimportmultiple.c +++ b/Modules/_testimportmultiple.c @@ -4,9 +4,7 @@ * foo, bar), only the first one is called the same as the compiled file. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x03020000 diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 4607a3faf17f74..7d277df164d3ec 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1475,6 +1475,17 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) } +static PyObject * +get_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + if (interp == NULL) { + return NULL; + } + return PyLong_FromLongLong(interp->id_refcount); +} + + static void _xid_capsule_destructor(PyObject *capsule) { @@ -1625,6 +1636,17 @@ get_type_module_name(PyObject *self, PyObject *type) } +#ifdef Py_GIL_DISABLED +static PyObject * +get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + uintptr_t tid = _Py_ThreadId(); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(tid)); + return PyLong_FromUnsignedLongLong(tid); +} +#endif + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1682,12 +1704,16 @@ static PyMethodDef module_functions[] = { {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, + {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF {"get_type_module_name", get_type_module_name, METH_O}, +#ifdef Py_GIL_DISABLED + {"py_thread_id", get_py_thread_id, METH_NOARGS}, +#endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 418f71c1441995..83081f73a72f64 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -372,6 +372,104 @@ test_lock_once(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +struct test_rwlock_data { + Py_ssize_t nthreads; + _PyRWMutex rw; + PyEvent step1; + PyEvent step2; + PyEvent step3; + PyEvent done; +}; + +static void +rdlock_thread(void *arg) +{ + struct test_rwlock_data *test_data = arg; + + // Acquire the lock in read mode + _PyRWMutex_RLock(&test_data->rw); + PyEvent_Wait(&test_data->step1); + _PyRWMutex_RUnlock(&test_data->rw); + + _PyRWMutex_RLock(&test_data->rw); + PyEvent_Wait(&test_data->step3); + _PyRWMutex_RUnlock(&test_data->rw); + + if (_Py_atomic_add_ssize(&test_data->nthreads, -1) == 1) { + _PyEvent_Notify(&test_data->done); + } +} +static void +wrlock_thread(void *arg) +{ + struct test_rwlock_data *test_data = arg; + + // First acquire the lock in write mode + _PyRWMutex_Lock(&test_data->rw); + PyEvent_Wait(&test_data->step2); + _PyRWMutex_Unlock(&test_data->rw); + + if (_Py_atomic_add_ssize(&test_data->nthreads, -1) == 1) { + _PyEvent_Notify(&test_data->done); + } +} + +static void +wait_until(uintptr_t *ptr, uintptr_t value) +{ + // wait up to two seconds for *ptr == value + int iters = 0; + uintptr_t bits; + do { + pysleep(10); + bits = _Py_atomic_load_uintptr(ptr); + iters++; + } while (bits != value && iters < 200); +} + +static PyObject * +test_lock_rwlock(PyObject *self, PyObject *obj) +{ + struct test_rwlock_data test_data = {.nthreads = 3}; + + _PyRWMutex_Lock(&test_data.rw); + assert(test_data.rw.bits == 1); + + _PyRWMutex_Unlock(&test_data.rw); + assert(test_data.rw.bits == 0); + + // Start two readers + PyThread_start_new_thread(rdlock_thread, &test_data); + PyThread_start_new_thread(rdlock_thread, &test_data); + + // wait up to two seconds for the threads to attempt to read-lock "rw" + wait_until(&test_data.rw.bits, 8); + assert(test_data.rw.bits == 8); + + // start writer (while readers hold lock) + PyThread_start_new_thread(wrlock_thread, &test_data); + wait_until(&test_data.rw.bits, 10); + assert(test_data.rw.bits == 10); + + // readers release lock, writer should acquire it + _PyEvent_Notify(&test_data.step1); + wait_until(&test_data.rw.bits, 3); + assert(test_data.rw.bits == 3); + + // writer releases lock, readers acquire it + _PyEvent_Notify(&test_data.step2); + wait_until(&test_data.rw.bits, 8); + assert(test_data.rw.bits == 8); + + // readers release lock again + _PyEvent_Notify(&test_data.step3); + wait_until(&test_data.rw.bits, 0); + assert(test_data.rw.bits == 0); + + PyEvent_Wait(&test_data.done); + Py_RETURN_NONE; +} + static PyMethodDef test_methods[] = { {"test_lock_basic", test_lock_basic, METH_NOARGS}, {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS}, @@ -380,6 +478,7 @@ static PyMethodDef test_methods[] = { _TESTINTERNALCAPI_BENCHMARK_LOCKS_METHODDEF {"test_lock_benchmark", test_lock_benchmark, METH_NOARGS}, {"test_lock_once", test_lock_once, METH_NOARGS}, + {"test_lock_rwlock", test_lock_rwlock, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index c608789b5fbd5c..afcf646e3bc19e 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1115,12 +1115,10 @@ local_getattro(localobject *self, PyObject *name) } /* Optimization: just look in dict ourselves */ - PyObject *value = PyDict_GetItemWithError(ldict, name); - if (value != NULL) { - return Py_NewRef(value); - } - if (PyErr_Occurred()) { - return NULL; + PyObject *value; + if (PyDict_GetItemRef(ldict, name, &value) != 0) { + // found or error + return value; } /* Fall back on generic to get __class__ and __dict__ */ diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index f9a18644945c65..f6181168a85ae1 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -735,8 +735,9 @@ newPyTclObject(Tcl_Obj *arg) } static void -PyTclObject_dealloc(PyTclObject *self) +PyTclObject_dealloc(PyObject *_self) { + PyTclObject *self = (PyTclObject *)_self; PyObject *tp = (PyObject *) Py_TYPE(self); Tcl_DecrRefCount(self->value); Py_XDECREF(self->string); @@ -749,8 +750,9 @@ PyDoc_STRVAR(PyTclObject_string__doc__, "the string representation of this object, either as str or bytes"); static PyObject * -PyTclObject_string(PyTclObject *self, void *ignored) +PyTclObject_string(PyObject *_self, void *ignored) { + PyTclObject *self = (PyTclObject *)_self; if (!self->string) { self->string = unicodeFromTclObj(self->value); if (!self->string) @@ -760,8 +762,9 @@ PyTclObject_string(PyTclObject *self, void *ignored) } static PyObject * -PyTclObject_str(PyTclObject *self) +PyTclObject_str(PyObject *_self) { + PyTclObject *self = (PyTclObject *)_self; if (self->string) { return Py_NewRef(self->string); } @@ -770,9 +773,10 @@ PyTclObject_str(PyTclObject *self) } static PyObject * -PyTclObject_repr(PyTclObject *self) +PyTclObject_repr(PyObject *_self) { - PyObject *repr, *str = PyTclObject_str(self); + PyTclObject *self = (PyTclObject *)_self; + PyObject *repr, *str = PyTclObject_str(_self); if (str == NULL) return NULL; repr = PyUnicode_FromFormat("<%s object: %R>", @@ -809,23 +813,24 @@ PyTclObject_richcompare(PyObject *self, PyObject *other, int op) PyDoc_STRVAR(get_typename__doc__, "name of the Tcl type"); static PyObject* -get_typename(PyTclObject* obj, void* ignored) +get_typename(PyObject *self, void* ignored) { + PyTclObject *obj = (PyTclObject *)self; return unicodeFromTclString(obj->value->typePtr->name); } static PyGetSetDef PyTclObject_getsetlist[] = { - {"typename", (getter)get_typename, NULL, get_typename__doc__}, - {"string", (getter)PyTclObject_string, NULL, + {"typename", get_typename, NULL, get_typename__doc__}, + {"string", PyTclObject_string, NULL, PyTclObject_string__doc__}, {0}, }; static PyType_Slot PyTclObject_Type_slots[] = { - {Py_tp_dealloc, (destructor)PyTclObject_dealloc}, - {Py_tp_repr, (reprfunc)PyTclObject_repr}, - {Py_tp_str, (reprfunc)PyTclObject_str}, + {Py_tp_dealloc, PyTclObject_dealloc}, + {Py_tp_repr, PyTclObject_repr}, + {Py_tp_str, PyTclObject_str}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_richcompare, PyTclObject_richcompare}, {Py_tp_getset, PyTclObject_getsetlist}, @@ -1202,7 +1207,7 @@ typedef struct Tkapp_CallEvent { Tcl_Condition *done; } Tkapp_CallEvent; -void +static void Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, int objc) { int i; @@ -1306,8 +1311,9 @@ Tkapp_ObjectResult(TkappObject *self) hold the Python lock. */ static int -Tkapp_CallProc(Tkapp_CallEvent *e, int flags) +Tkapp_CallProc(Tcl_Event *evPtr, int flags) { + Tkapp_CallEvent *e = (Tkapp_CallEvent *)evPtr; Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv; int objc; @@ -1385,7 +1391,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) PyErr_NoMemory(); return NULL; } - ev->ev.proc = (Tcl_EventProc*)Tkapp_CallProc; + ev->ev.proc = Tkapp_CallProc; ev->self = self; ev->args = args; ev->res = &res; @@ -1624,8 +1630,9 @@ var_perform(VarEvent *ev) } static int -var_proc(VarEvent* ev, int flags) +var_proc(Tcl_Event *evPtr, int flags) { + VarEvent *ev = (VarEvent *)evPtr; ENTER_PYTHON var_perform(ev); Tcl_MutexLock(&var_mutex); @@ -1663,7 +1670,7 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags) ev->res = &res; ev->exc = &exc; ev->cond = &cond; - ev->ev.proc = (Tcl_EventProc*)var_proc; + ev->ev.proc = var_proc; Tkapp_ThreadSend(self, (Tcl_Event*)ev, &cond, &var_mutex); Tcl_ConditionFinalize(&cond); if (!res) { @@ -2236,8 +2243,9 @@ typedef struct CommandEvent{ } CommandEvent; static int -Tkapp_CommandProc(CommandEvent *ev, int flags) +Tkapp_CommandProc(Tcl_Event *evPtr, int flags) { + CommandEvent *ev = (CommandEvent *)evPtr; if (ev->create) *ev->status = Tcl_CreateObjCommand( ev->interp, ev->name, PythonCmd, @@ -2290,7 +2298,7 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, PyMem_Free(data); return NULL; } - ev->ev.proc = (Tcl_EventProc*)Tkapp_CommandProc; + ev->ev.proc = Tkapp_CommandProc; ev->interp = self->interp; ev->create = 1; ev->name = name; @@ -2343,7 +2351,7 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) PyErr_NoMemory(); return NULL; } - ev->ev.proc = (Tcl_EventProc*)Tkapp_CommandProc; + ev->ev.proc = Tkapp_CommandProc; ev->interp = self->interp; ev->create = 0; ev->name = name; diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index d8b211c632eef1..4b6852c0d0ec73 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,9 +3,7 @@ * DCE compatible Universally Unique Identifier library. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 1c9ae3b87adf7c..4e9b8a82a3f630 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -8,7 +8,6 @@ #include "Python.h" #include "interpreteridobject.h" #include "pycore_crossinterp.h" // struct _xid -#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_interp.h" // _PyInterpreterState_LookUpID() #ifdef MS_WINDOWS @@ -263,136 +262,6 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) } -/* Cross-interpreter Buffer Views *******************************************/ - -// XXX Release when the original interpreter is destroyed. - -typedef struct { - PyObject_HEAD - Py_buffer *view; - int64_t interpid; -} XIBufferViewObject; - -static PyObject * -xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) -{ - assert(data->data != NULL); - assert(data->obj == NULL); - assert(data->interpid >= 0); - XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); - if (self == NULL) { - return NULL; - } - PyObject_Init((PyObject *)self, cls); - self->view = (Py_buffer *)data->data; - self->interpid = data->interpid; - return (PyObject *)self; -} - -static void -xibufferview_dealloc(XIBufferViewObject *self) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interpid); - /* If the interpreter is no longer alive then we have problems, - since other objects may be using the buffer still. */ - assert(interp != NULL); - - if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { - // XXX Emit a warning? - PyErr_Clear(); - } - - PyTypeObject *tp = Py_TYPE(self); - tp->tp_free(self); - /* "Instances of heap-allocated types hold a reference to their type." - * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol - * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse - */ - // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, - // like we do for _abc._abc_data? - Py_DECREF(tp); -} - -static int -xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags) -{ - /* Only PyMemoryView_FromObject() should ever call this, - via _memoryview_from_xid() below. */ - *view = *self->view; - view->obj = (PyObject *)self; - // XXX Should we leave it alone? - view->internal = NULL; - return 0; -} - -static PyType_Slot XIBufferViewType_slots[] = { - {Py_tp_dealloc, (destructor)xibufferview_dealloc}, - {Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf}, - // We don't bother with Py_bf_releasebuffer since we don't need it. - {0, NULL}, -}; - -static PyType_Spec XIBufferViewType_spec = { - .name = MODULE_NAME ".CrossInterpreterBufferView", - .basicsize = sizeof(XIBufferViewObject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), - .slots = XIBufferViewType_slots, -}; - - -/* extra XID types **********************************************************/ - -static PyTypeObject * _get_current_xibufferview_type(void); - -static PyObject * -_memoryview_from_xid(_PyCrossInterpreterData *data) -{ - PyTypeObject *cls = _get_current_xibufferview_type(); - if (cls == NULL) { - return NULL; - } - PyObject *obj = xibufferview_from_xid(cls, data); - if (obj == NULL) { - return NULL; - } - return PyMemoryView_FromObject(obj); -} - -static int -_memoryview_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); - if (view == NULL) { - return -1; - } - if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) { - PyMem_RawFree(view); - return -1; - } - _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, - _memoryview_from_xid); - return 0; -} - -static int -register_builtin_xid_types(struct xid_class_registry *classes) -{ - PyTypeObject *cls; - crossinterpdatafunc func; - - // builtin memoryview - cls = &PyMemoryView_Type; - func = _memoryview_shared; - if (register_xid_class(cls, func, classes)) { - return -1; - } - - return 0; -} - - /* module state *************************************************************/ typedef struct { @@ -405,7 +274,6 @@ typedef struct { /* heap types */ PyTypeObject *ChannelInfoType; PyTypeObject *ChannelIDType; - PyTypeObject *XIBufferViewType; /* exceptions */ PyObject *ChannelError; @@ -449,7 +317,6 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) /* heap types */ Py_VISIT(state->ChannelInfoType); Py_VISIT(state->ChannelIDType); - Py_VISIT(state->XIBufferViewType); /* exceptions */ Py_VISIT(state->ChannelError); @@ -474,7 +341,6 @@ clear_module_state(module_state *state) (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); } Py_CLEAR(state->ChannelIDType); - Py_CLEAR(state->XIBufferViewType); /* exceptions */ Py_CLEAR(state->ChannelError); @@ -487,17 +353,6 @@ clear_module_state(module_state *state) } -static PyTypeObject * -_get_current_xibufferview_type(void) -{ - module_state *state = _get_current_module_state(); - if (state == NULL) { - return NULL; - } - return state->XIBufferViewType; -} - - /* channel-specific code ****************************************************/ #define CHANNEL_SEND 1 @@ -2774,10 +2629,11 @@ _get_current_channelend_type(int end) cls = state->recv_channel_type; } if (cls == NULL) { - PyObject *highlevel = PyImport_ImportModule("interpreters"); + // Force the module to be loaded, to register the type. + PyObject *highlevel = PyImport_ImportModule("interpreters.channel"); if (highlevel == NULL) { PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters"); + highlevel = PyImport_ImportModule("test.support.interpreters.channel"); if (highlevel == NULL) { return NULL; } @@ -3463,18 +3319,6 @@ module_exec(PyObject *mod) goto error; } - // XIBufferView - state->XIBufferViewType = add_new_type(mod, &XIBufferViewType_spec, NULL, - xid_classes); - if (state->XIBufferViewType == NULL) { - goto error; - } - - // Register external types. - if (register_builtin_xid_types(xid_classes) < 0) { - goto error; - } - /* Make sure chnnels drop objects owned by this interpreter. */ PyInterpreterState *interp = _get_current_interp(); PyUnstable_AtExit(interp, clear_interpreter, (void *)interp); diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c new file mode 100644 index 00000000000000..537ba9188055dd --- /dev/null +++ b/Modules/_xxinterpqueuesmodule.c @@ -0,0 +1,1687 @@ +/* interpreters module */ +/* low-level access to interpreter primitives */ + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include "pycore_crossinterp.h" // struct _xid + + +#define MODULE_NAME "_xxinterpqueues" + + +#define GLOBAL_MALLOC(TYPE) \ + PyMem_RawMalloc(sizeof(TYPE)) +#define GLOBAL_FREE(VAR) \ + PyMem_RawFree(VAR) + + +#define XID_IGNORE_EXC 1 +#define XID_FREE 2 + +static int +_release_xid_data(_PyCrossInterpreterData *data, int flags) +{ + int ignoreexc = flags & XID_IGNORE_EXC; + PyObject *exc; + if (ignoreexc) { + exc = PyErr_GetRaisedException(); + } + int res; + if (flags & XID_FREE) { + res = _PyCrossInterpreterData_ReleaseAndRawFree(data); + } + else { + res = _PyCrossInterpreterData_Release(data); + } + if (res < 0) { + /* The owning interpreter is already destroyed. */ + if (ignoreexc) { + // XXX Emit a warning? + PyErr_Clear(); + } + } + if (flags & XID_FREE) { + /* Either way, we free the data. */ + } + if (ignoreexc) { + PyErr_SetRaisedException(exc); + } + return res; +} + + +static PyInterpreterState * +_get_current_interp(void) +{ + // PyInterpreterState_Get() aborts if lookup fails, so don't need + // to check the result for NULL. + return PyInterpreterState_Get(); +} + +static PyObject * +_get_current_module(void) +{ + PyObject *name = PyUnicode_FromString(MODULE_NAME); + if (name == NULL) { + return NULL; + } + PyObject *mod = PyImport_GetModule(name); + Py_DECREF(name); + if (mod == NULL) { + return NULL; + } + assert(mod != Py_None); + return mod; +} + + +struct idarg_int64_converter_data { + // input: + const char *label; + // output: + int64_t id; +}; + +static int +idarg_int64_converter(PyObject *arg, void *ptr) +{ + int64_t id; + struct idarg_int64_converter_data *data = ptr; + + const char *label = data->label; + if (label == NULL) { + label = "ID"; + } + + if (PyIndex_Check(arg)) { + int overflow = 0; + id = PyLong_AsLongLongAndOverflow(arg, &overflow); + if (id == -1 && PyErr_Occurred()) { + return 0; + } + else if (id == -1 && overflow == 1) { + PyErr_Format(PyExc_OverflowError, + "max %s is %lld, got %R", label, INT64_MAX, arg); + return 0; + } + else if (id < 0) { + PyErr_Format(PyExc_ValueError, + "%s must be a non-negative int, got %R", label, arg); + return 0; + } + } + else { + PyErr_Format(PyExc_TypeError, + "%s must be an int, got %.100s", + label, Py_TYPE(arg)->tp_name); + return 0; + } + data->id = id; + return 1; +} + + +/* module state *************************************************************/ + +typedef struct { + /* external types (added at runtime by interpreters module) */ + PyTypeObject *queue_type; + + /* QueueError (and its subclasses) */ + PyObject *QueueError; + PyObject *QueueNotFoundError; + PyObject *QueueEmpty; + PyObject *QueueFull; +} module_state; + +static inline module_state * +get_module_state(PyObject *mod) +{ + assert(mod != NULL); + module_state *state = PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static int +traverse_module_state(module_state *state, visitproc visit, void *arg) +{ + /* external types */ + Py_VISIT(state->queue_type); + + /* QueueError */ + Py_VISIT(state->QueueError); + Py_VISIT(state->QueueNotFoundError); + Py_VISIT(state->QueueEmpty); + Py_VISIT(state->QueueFull); + + return 0; +} + +static int +clear_module_state(module_state *state) +{ + /* external types */ + Py_CLEAR(state->queue_type); + + /* QueueError */ + Py_CLEAR(state->QueueError); + Py_CLEAR(state->QueueNotFoundError); + Py_CLEAR(state->QueueEmpty); + Py_CLEAR(state->QueueFull); + + return 0; +} + + +/* error codes **************************************************************/ + +#define ERR_EXCEPTION_RAISED (-1) +// multi-queue errors +#define ERR_QUEUES_ALLOC (-11) +#define ERR_QUEUE_ALLOC (-12) +#define ERR_NO_NEXT_QUEUE_ID (-13) +#define ERR_QUEUE_NOT_FOUND (-14) +// single-queue errors +#define ERR_QUEUE_EMPTY (-21) +#define ERR_QUEUE_FULL (-22) + +static int +resolve_module_errcode(module_state *state, int errcode, int64_t qid, + PyObject **p_exctype, PyObject **p_msgobj) +{ + PyObject *exctype = NULL; + PyObject *msg = NULL; + switch (errcode) { + case ERR_NO_NEXT_QUEUE_ID: + exctype = state->QueueError; + msg = PyUnicode_FromString("ran out of queue IDs"); + break; + case ERR_QUEUE_NOT_FOUND: + exctype = state->QueueNotFoundError; + msg = PyUnicode_FromFormat("queue %" PRId64 " not found", qid); + break; + case ERR_QUEUE_EMPTY: + exctype = state->QueueEmpty; + msg = PyUnicode_FromFormat("queue %" PRId64 " is empty", qid); + break; + case ERR_QUEUE_FULL: + exctype = state->QueueFull; + msg = PyUnicode_FromFormat("queue %" PRId64 " is full", qid); + break; + default: + PyErr_Format(PyExc_ValueError, + "unsupported error code %d", errcode); + return -1; + } + + if (msg == NULL) { + assert(PyErr_Occurred()); + return -1; + } + *p_exctype = exctype; + *p_msgobj = msg; + return 0; +} + + +/* QueueError ***************************************************************/ + +static int +add_exctype(PyObject *mod, PyObject **p_state_field, + const char *qualname, const char *doc, PyObject *base) +{ +#ifndef NDEBUG + const char *dot = strrchr(qualname, '.'); + assert(dot != NULL); + const char *name = dot+1; + assert(*p_state_field == NULL); + assert(!PyObject_HasAttrStringWithError(mod, name)); +#endif + PyObject *exctype = PyErr_NewExceptionWithDoc(qualname, doc, base, NULL); + if (exctype == NULL) { + return -1; + } + if (PyModule_AddType(mod, (PyTypeObject *)exctype) < 0) { + Py_DECREF(exctype); + return -1; + } + *p_state_field = exctype; + return 0; +} + +static int +add_QueueError(PyObject *mod) +{ + module_state *state = get_module_state(mod); + +#define PREFIX "test.support.interpreters." +#define ADD_EXCTYPE(NAME, BASE, DOC) \ + if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ + return -1; \ + } + ADD_EXCTYPE(QueueError, PyExc_RuntimeError, + "Indicates that a queue-related error happened.") + ADD_EXCTYPE(QueueNotFoundError, state->QueueError, NULL) + ADD_EXCTYPE(QueueEmpty, state->QueueError, NULL) + ADD_EXCTYPE(QueueFull, state->QueueError, NULL) +#undef ADD_EXCTYPE +#undef PREFIX + + return 0; +} + +static int +handle_queue_error(int err, PyObject *mod, int64_t qid) +{ + if (err == 0) { + assert(!PyErr_Occurred()); + return 0; + } + assert(err < 0); + assert((err == -1) == (PyErr_Occurred() != NULL)); + + module_state *state; + switch (err) { + case ERR_QUEUE_ALLOC: // fall through + case ERR_QUEUES_ALLOC: + PyErr_NoMemory(); + break; + default: + state = get_module_state(mod); + assert(state->QueueError != NULL); + PyObject *exctype = NULL; + PyObject *msg = NULL; + if (resolve_module_errcode(state, err, qid, &exctype, &msg) < 0) { + return -1; + } + PyObject *exc = PyObject_CallOneArg(exctype, msg); + Py_DECREF(msg); + if (exc == NULL) { + return -1; + } + PyErr_SetObject(exctype, exc); + Py_DECREF(exc); + } + return 1; +} + + +/* the basic queue **********************************************************/ + +struct _queueitem; + +typedef struct _queueitem { + _PyCrossInterpreterData *data; + struct _queueitem *next; +} _queueitem; + +static void +_queueitem_init(_queueitem *item, _PyCrossInterpreterData *data) +{ + *item = (_queueitem){ + .data = data, + }; +} + +static void +_queueitem_clear(_queueitem *item) +{ + item->next = NULL; + + if (item->data != NULL) { + // It was allocated in queue_put(). + (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + item->data = NULL; + } +} + +static _queueitem * +_queueitem_new(_PyCrossInterpreterData *data) +{ + _queueitem *item = GLOBAL_MALLOC(_queueitem); + if (item == NULL) { + PyErr_NoMemory(); + return NULL; + } + _queueitem_init(item, data); + return item; +} + +static void +_queueitem_free(_queueitem *item) +{ + _queueitem_clear(item); + GLOBAL_FREE(item); +} + +static void +_queueitem_free_all(_queueitem *item) +{ + while (item != NULL) { + _queueitem *last = item; + item = item->next; + _queueitem_free(last); + } +} + +static void +_queueitem_popped(_queueitem *item, _PyCrossInterpreterData **p_data) +{ + *p_data = item->data; + // We clear them here, so they won't be released in _queueitem_clear(). + item->data = NULL; + _queueitem_free(item); +} + + +/* the queue */ +typedef struct _queue { + Py_ssize_t num_waiters; // protected by global lock + PyThread_type_lock mutex; + int alive; + struct _queueitems { + Py_ssize_t maxsize; + Py_ssize_t count; + _queueitem *first; + _queueitem *last; + } items; +} _queue; + +static int +_queue_init(_queue *queue, Py_ssize_t maxsize) +{ + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_QUEUE_ALLOC; + } + *queue = (_queue){ + .mutex = mutex, + .alive = 1, + .items = { + .maxsize = maxsize, + }, + }; + return 0; +} + +static void +_queue_clear(_queue *queue) +{ + assert(!queue->alive); + assert(queue->num_waiters == 0); + _queueitem_free_all(queue->items.first); + assert(queue->mutex != NULL); + PyThread_free_lock(queue->mutex); + *queue = (_queue){0}; +} + +static void +_queue_kill_and_wait(_queue *queue) +{ + // Mark it as dead. + PyThread_acquire_lock(queue->mutex, WAIT_LOCK); + assert(queue->alive); + queue->alive = 0; + PyThread_release_lock(queue->mutex); + + // Wait for all waiters to fail. + while (queue->num_waiters > 0) { + PyThread_acquire_lock(queue->mutex, WAIT_LOCK); + PyThread_release_lock(queue->mutex); + }; +} + +static void +_queue_mark_waiter(_queue *queue, PyThread_type_lock parent_mutex) +{ + if (parent_mutex != NULL) { + PyThread_acquire_lock(parent_mutex, WAIT_LOCK); + queue->num_waiters += 1; + PyThread_release_lock(parent_mutex); + } + else { + // The caller must be holding the parent lock already. + queue->num_waiters += 1; + } +} + +static void +_queue_unmark_waiter(_queue *queue, PyThread_type_lock parent_mutex) +{ + if (parent_mutex != NULL) { + PyThread_acquire_lock(parent_mutex, WAIT_LOCK); + queue->num_waiters -= 1; + PyThread_release_lock(parent_mutex); + } + else { + // The caller must be holding the parent lock already. + queue->num_waiters -= 1; + } +} + +static int +_queue_lock(_queue *queue) +{ + // The queue must be marked as a waiter already. + PyThread_acquire_lock(queue->mutex, WAIT_LOCK); + if (!queue->alive) { + PyThread_release_lock(queue->mutex); + return ERR_QUEUE_NOT_FOUND; + } + return 0; +} + +static void +_queue_unlock(_queue *queue) +{ + PyThread_release_lock(queue->mutex); +} + +static int +_queue_add(_queue *queue, _PyCrossInterpreterData *data) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + Py_ssize_t maxsize = queue->items.maxsize; + if (maxsize <= 0) { + maxsize = PY_SSIZE_T_MAX; + } + if (queue->items.count >= maxsize) { + _queue_unlock(queue); + return ERR_QUEUE_FULL; + } + + _queueitem *item = _queueitem_new(data); + if (item == NULL) { + _queue_unlock(queue); + return -1; + } + + queue->items.count += 1; + if (queue->items.first == NULL) { + queue->items.first = item; + } + else { + queue->items.last->next = item; + } + queue->items.last = item; + + _queue_unlock(queue); + return 0; +} + +static int +_queue_next(_queue *queue, _PyCrossInterpreterData **p_data) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + assert(queue->items.count >= 0); + _queueitem *item = queue->items.first; + if (item == NULL) { + _queue_unlock(queue); + return ERR_QUEUE_EMPTY; + } + queue->items.first = item->next; + if (queue->items.last == item) { + queue->items.last = NULL; + } + queue->items.count -= 1; + + _queueitem_popped(item, p_data); + + _queue_unlock(queue); + return 0; +} + +static int +_queue_get_maxsize(_queue *queue, Py_ssize_t *p_maxsize) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + *p_maxsize = queue->items.maxsize; + + _queue_unlock(queue); + return 0; +} + +static int +_queue_is_full(_queue *queue, int *p_is_full) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + assert(queue->items.count <= queue->items.maxsize); + *p_is_full = queue->items.count == queue->items.maxsize; + + _queue_unlock(queue); + return 0; +} + +static int +_queue_get_count(_queue *queue, Py_ssize_t *p_count) +{ + int err = _queue_lock(queue); + if (err < 0) { + return err; + } + + *p_count = queue->items.count; + + _queue_unlock(queue); + return 0; +} + +static void +_queue_clear_interpreter(_queue *queue, int64_t interpid) +{ + int err = _queue_lock(queue); + if (err == ERR_QUEUE_NOT_FOUND) { + // The queue is already destroyed, so there's nothing to clear. + assert(!PyErr_Occurred()); + return; + } + assert(err == 0); // There should be no other errors. + + _queueitem *prev = NULL; + _queueitem *next = queue->items.first; + while (next != NULL) { + _queueitem *item = next; + next = item->next; + if (item->data->interpid == interpid) { + if (prev == NULL) { + queue->items.first = item->next; + } + else { + prev->next = item->next; + } + _queueitem_free(item); + queue->items.count -= 1; + } + else { + prev = item; + } + } + + _queue_unlock(queue); +} + + +/* external queue references ************************************************/ + +struct _queueref; + +typedef struct _queueref { + struct _queueref *next; + int64_t qid; + Py_ssize_t refcount; + _queue *queue; +} _queueref; + +static _queueref * +_queuerefs_find(_queueref *first, int64_t qid, _queueref **pprev) +{ + _queueref *prev = NULL; + _queueref *ref = first; + while (ref != NULL) { + if (ref->qid == qid) { + break; + } + prev = ref; + ref = ref->next; + } + if (pprev != NULL) { + *pprev = prev; + } + return ref; +} + + +/* a collection of queues ***************************************************/ + +typedef struct _queues { + PyThread_type_lock mutex; + _queueref *head; + int64_t count; + int64_t next_id; +} _queues; + +static void +_queues_init(_queues *queues, PyThread_type_lock mutex) +{ + queues->mutex = mutex; + queues->head = NULL; + queues->count = 0; + queues->next_id = 1; +} + +static void +_queues_fini(_queues *queues) +{ + assert(queues->count == 0); + assert(queues->head == NULL); + if (queues->mutex != NULL) { + PyThread_free_lock(queues->mutex); + queues->mutex = NULL; + } +} + +static int64_t +_queues_next_id(_queues *queues) // needs lock +{ + int64_t qid = queues->next_id; + if (qid < 0) { + /* overflow */ + return ERR_NO_NEXT_QUEUE_ID; + } + queues->next_id += 1; + return qid; +} + +static int +_queues_lookup(_queues *queues, int64_t qid, _queue **res) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *ref = _queuerefs_find(queues->head, qid, NULL); + if (ref == NULL) { + PyThread_release_lock(queues->mutex); + return ERR_QUEUE_NOT_FOUND; + } + assert(ref->queue != NULL); + _queue *queue = ref->queue; + _queue_mark_waiter(queue, NULL); + // The caller must unmark it. + + PyThread_release_lock(queues->mutex); + + *res = queue; + return 0; +} + +static int64_t +_queues_add(_queues *queues, _queue *queue) +{ + int64_t qid = -1; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + // Create a new ref. + int64_t _qid = _queues_next_id(queues); + if (_qid < 0) { + goto done; + } + _queueref *ref = GLOBAL_MALLOC(_queueref); + if (ref == NULL) { + qid = ERR_QUEUE_ALLOC; + goto done; + } + *ref = (_queueref){ + .qid = _qid, + .queue = queue, + }; + + // Add it to the list. + // We assume that the queue is a new one (not already in the list). + ref->next = queues->head; + queues->head = ref; + queues->count += 1; + + qid = _qid; +done: + PyThread_release_lock(queues->mutex); + return qid; +} + +static void +_queues_remove_ref(_queues *queues, _queueref *ref, _queueref *prev, + _queue **p_queue) +{ + assert(ref->queue != NULL); + + if (ref == queues->head) { + queues->head = ref->next; + } + else { + prev->next = ref->next; + } + ref->next = NULL; + queues->count -= 1; + + *p_queue = ref->queue; + ref->queue = NULL; + GLOBAL_FREE(ref); +} + +static int +_queues_remove(_queues *queues, int64_t qid, _queue **p_queue) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *prev = NULL; + _queueref *ref = _queuerefs_find(queues->head, qid, &prev); + if (ref == NULL) { + PyThread_release_lock(queues->mutex); + return ERR_QUEUE_NOT_FOUND; + } + + _queues_remove_ref(queues, ref, prev, p_queue); + PyThread_release_lock(queues->mutex); + + return 0; +} + +static int +_queues_incref(_queues *queues, int64_t qid) +{ + // XXX Track interpreter IDs? + int res = -1; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *ref = _queuerefs_find(queues->head, qid, NULL); + if (ref == NULL) { + assert(!PyErr_Occurred()); + res = ERR_QUEUE_NOT_FOUND; + goto done; + } + ref->refcount += 1; + + res = 0; +done: + PyThread_release_lock(queues->mutex); + return res; +} + +static void _queue_free(_queue *); + +static void +_queues_decref(_queues *queues, int64_t qid) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *prev = NULL; + _queueref *ref = _queuerefs_find(queues->head, qid, &prev); + if (ref == NULL) { + assert(!PyErr_Occurred()); + // Already destroyed. + // XXX Warn? + goto finally; + } + assert(ref->refcount > 0); + ref->refcount -= 1; + + // Destroy if no longer used. + assert(ref->queue != NULL); + if (ref->refcount == 0) { + _queue *queue = NULL; + _queues_remove_ref(queues, ref, prev, &queue); + PyThread_release_lock(queues->mutex); + + _queue_kill_and_wait(queue); + _queue_free(queue); + return; + } + +finally: + PyThread_release_lock(queues->mutex); +} + +static int64_t * +_queues_list_all(_queues *queues, int64_t *count) +{ + int64_t *qids = NULL; + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(queues->count)); + if (ids == NULL) { + goto done; + } + _queueref *ref = queues->head; + for (int64_t i=0; ref != NULL; ref = ref->next, i++) { + ids[i] = ref->qid; + } + *count = queues->count; + + qids = ids; +done: + PyThread_release_lock(queues->mutex); + return qids; +} + +static void +_queues_clear_interpreter(_queues *queues, int64_t interpid) +{ + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + + _queueref *ref = queues->head; + for (; ref != NULL; ref = ref->next) { + assert(ref->queue != NULL); + _queue_clear_interpreter(ref->queue, interpid); + } + + PyThread_release_lock(queues->mutex); +} + + +/* "high"-level queue-related functions *************************************/ + +static void +_queue_free(_queue *queue) +{ + _queue_clear(queue); + GLOBAL_FREE(queue); +} + +// Create a new queue. +static int64_t +queue_create(_queues *queues, Py_ssize_t maxsize) +{ + _queue *queue = GLOBAL_MALLOC(_queue); + if (queue == NULL) { + return ERR_QUEUE_ALLOC; + } + int err = _queue_init(queue, maxsize); + if (err < 0) { + GLOBAL_FREE(queue); + return (int64_t)err; + } + int64_t qid = _queues_add(queues, queue); + if (qid < 0) { + _queue_clear(queue); + GLOBAL_FREE(queue); + } + return qid; +} + +// Completely destroy the queue. +static int +queue_destroy(_queues *queues, int64_t qid) +{ + _queue *queue = NULL; + int err = _queues_remove(queues, qid, &queue); + if (err < 0) { + return err; + } + _queue_kill_and_wait(queue); + _queue_free(queue); + return 0; +} + +// Push an object onto the queue. +static int +queue_put(_queues *queues, int64_t qid, PyObject *obj) +{ + // Look up the queue. + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err != 0) { + return err; + } + assert(queue != NULL); + + // Convert the object to cross-interpreter data. + _PyCrossInterpreterData *data = GLOBAL_MALLOC(_PyCrossInterpreterData); + if (data == NULL) { + _queue_unmark_waiter(queue, queues->mutex); + return -1; + } + if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + _queue_unmark_waiter(queue, queues->mutex); + GLOBAL_FREE(data); + return -1; + } + + // Add the data to the queue. + int res = _queue_add(queue, data); + _queue_unmark_waiter(queue, queues->mutex); + if (res != 0) { + // We may chain an exception here: + (void)_release_xid_data(data, 0); + GLOBAL_FREE(data); + return res; + } + + return 0; +} + +// Pop the next object off the queue. Fail if empty. +// XXX Support a "wait" mutex? +static int +queue_get(_queues *queues, int64_t qid, PyObject **res) +{ + int err; + *res = NULL; + + // Look up the queue. + _queue *queue = NULL; + err = _queues_lookup(queues, qid, &queue); + if (err != 0) { + return err; + } + // Past this point we are responsible for releasing the mutex. + assert(queue != NULL); + + // Pop off the next item from the queue. + _PyCrossInterpreterData *data = NULL; + err = _queue_next(queue, &data); + _queue_unmark_waiter(queue, queues->mutex); + if (err != 0) { + return err; + } + else if (data == NULL) { + assert(!PyErr_Occurred()); + return 0; + } + + // Convert the data back to an object. + PyObject *obj = _PyCrossInterpreterData_NewObject(data); + if (obj == NULL) { + assert(PyErr_Occurred()); + // It was allocated in queue_put(), so we free it. + (void)_release_xid_data(data, XID_IGNORE_EXC | XID_FREE); + return -1; + } + // It was allocated in queue_put(), so we free it. + int release_res = _release_xid_data(data, XID_FREE); + if (release_res < 0) { + // The source interpreter has been destroyed already. + assert(PyErr_Occurred()); + Py_DECREF(obj); + return -1; + } + + *res = obj; + return 0; +} + +static int +queue_get_maxsize(_queues *queues, int64_t qid, Py_ssize_t *p_maxsize) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err < 0) { + return err; + } + err = _queue_get_maxsize(queue, p_maxsize); + _queue_unmark_waiter(queue, queues->mutex); + return err; +} + +static int +queue_is_full(_queues *queues, int64_t qid, int *p_is_full) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err < 0) { + return err; + } + err = _queue_is_full(queue, p_is_full); + _queue_unmark_waiter(queue, queues->mutex); + return err; +} + +static int +queue_get_count(_queues *queues, int64_t qid, Py_ssize_t *p_count) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err < 0) { + return err; + } + err = _queue_get_count(queue, p_count); + _queue_unmark_waiter(queue, queues->mutex); + return err; +} + + +/* external Queue objects ***************************************************/ + +static int _queueobj_shared(PyThreadState *, + PyObject *, _PyCrossInterpreterData *); + +static int +set_external_queue_type(PyObject *module, PyTypeObject *queue_type) +{ + module_state *state = get_module_state(module); + + if (state->queue_type != NULL) { + PyErr_SetString(PyExc_TypeError, "already registered"); + return -1; + } + state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); + + if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0) { + return -1; + } + + return 0; +} + +static PyTypeObject * +get_external_queue_type(PyObject *module) +{ + module_state *state = get_module_state(module); + + PyTypeObject *cls = state->queue_type; + if (cls == NULL) { + // Force the module to be loaded, to register the type. + PyObject *highlevel = PyImport_ImportModule("interpreters.queue"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.interpreters.queue"); + if (highlevel == NULL) { + return NULL; + } + } + Py_DECREF(highlevel); + cls = state->queue_type; + assert(cls != NULL); + } + return cls; +} + + +// XXX Use a new __xid__ protocol instead? + +struct _queueid_xid { + int64_t qid; +}; + +static _queues * _get_global_queues(void); + +static void * +_queueid_xid_new(int64_t qid) +{ + _queues *queues = _get_global_queues(); + if (_queues_incref(queues, qid) < 0) { + return NULL; + } + + struct _queueid_xid *data = PyMem_RawMalloc(sizeof(struct _queueid_xid)); + if (data == NULL) { + _queues_incref(queues, qid); + return NULL; + } + data->qid = qid; + return (void *)data; +} + +static void +_queueid_xid_free(void *data) +{ + int64_t qid = ((struct _queueid_xid *)data)->qid; + PyMem_RawFree(data); + _queues *queues = _get_global_queues(); + _queues_decref(queues, qid); +} + +static PyObject * +_queueobj_from_xid(_PyCrossInterpreterData *data) +{ + int64_t qid = *(int64_t *)data->data; + PyObject *qidobj = PyLong_FromLongLong(qid); + if (qidobj == NULL) { + return NULL; + } + + PyObject *mod = _get_current_module(); + if (mod == NULL) { + // XXX import it? + PyErr_SetString(PyExc_RuntimeError, + MODULE_NAME " module not imported yet"); + return NULL; + } + + PyTypeObject *cls = get_external_queue_type(mod); + Py_DECREF(mod); + if (cls == NULL) { + Py_DECREF(qidobj); + return NULL; + } + PyObject *obj = PyObject_CallOneArg((PyObject *)cls, (PyObject *)qidobj); + Py_DECREF(qidobj); + return obj; +} + +static int +_queueobj_shared(PyThreadState *tstate, PyObject *queueobj, + _PyCrossInterpreterData *data) +{ + PyObject *qidobj = PyObject_GetAttrString(queueobj, "_id"); + if (qidobj == NULL) { + return -1; + } + struct idarg_int64_converter_data converted = { + .label = "queue ID", + }; + int res = idarg_int64_converter(qidobj, &converted); + Py_DECREF(qidobj); + if (!res) { + assert(PyErr_Occurred()); + return -1; + } + + void *raw = _queueid_xid_new(converted.id); + if (raw == NULL) { + Py_DECREF(qidobj); + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL, + _queueobj_from_xid); + Py_DECREF(qidobj); + data->free = _queueid_xid_free; + return 0; +} + + +/* module level code ********************************************************/ + +/* globals is the process-global state for the module. It holds all + the data that we need to share between interpreters, so it cannot + hold PyObject values. */ +static struct globals { + int module_count; + _queues queues; +} _globals = {0}; + +static int +_globals_init(void) +{ + // XXX This isn't thread-safe. + _globals.module_count++; + if (_globals.module_count > 1) { + // Already initialized. + return 0; + } + + assert(_globals.queues.mutex == NULL); + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_QUEUES_ALLOC; + } + _queues_init(&_globals.queues, mutex); + return 0; +} + +static void +_globals_fini(void) +{ + // XXX This isn't thread-safe. + _globals.module_count--; + if (_globals.module_count > 0) { + return; + } + + _queues_fini(&_globals.queues); +} + +static _queues * +_get_global_queues(void) +{ + return &_globals.queues; +} + + +static void +clear_interpreter(void *data) +{ + if (_globals.module_count == 0) { + return; + } + PyInterpreterState *interp = (PyInterpreterState *)data; + assert(interp == _get_current_interp()); + int64_t interpid = PyInterpreterState_GetID(interp); + _queues_clear_interpreter(&_globals.queues, interpid); +} + + +typedef struct idarg_int64_converter_data qidarg_converter_data; + +static int +qidarg_converter(PyObject *arg, void *ptr) +{ + qidarg_converter_data *data = ptr; + if (data->label == NULL) { + data->label = "queue ID"; + } + return idarg_int64_converter(arg, ptr); +} + + +static PyObject * +queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"maxsize", NULL}; + Py_ssize_t maxsize = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n:create", kwlist, + &maxsize)) { + return NULL; + } + + int64_t qid = queue_create(&_globals.queues, maxsize); + if (qid < 0) { + (void)handle_queue_error((int)qid, self, qid); + return NULL; + } + + PyObject *qidobj = PyLong_FromLongLong(qid); + if (qidobj == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + int err = queue_destroy(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + // XXX issue a warning? + PyErr_Clear(); + } + PyErr_SetRaisedException(exc); + return NULL; + } + + return qidobj; +} + +PyDoc_STRVAR(queuesmod_create_doc, +"create() -> qid\n\ +\n\ +Create a new cross-interpreter queue and return its unique generated ID.\n\ +It is a new reference as though bind() had been called on the queue."); + +static PyObject * +queuesmod_destroy(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:destroy", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + int err = queue_destroy(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_destroy_doc, +"destroy(qid)\n\ +\n\ +Clear and destroy the queue. Afterward attempts to use the queue\n\ +will behave as though it never existed."); + +static PyObject * +queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t count = 0; + int64_t *qids = _queues_list_all(&_globals.queues, &count); + if (qids == NULL) { + if (count == 0) { + return PyList_New(0); + } + return NULL; + } + PyObject *ids = PyList_New((Py_ssize_t)count); + if (ids == NULL) { + goto finally; + } + int64_t *cur = qids; + for (int64_t i=0; i < count; cur++, i++) { + PyObject *qidobj = PyLong_FromLongLong(*cur); + if (qidobj == NULL) { + Py_SETREF(ids, NULL); + break; + } + PyList_SET_ITEM(ids, (Py_ssize_t)i, qidobj); + } + +finally: + PyMem_Free(qids); + return ids; +} + +PyDoc_STRVAR(queuesmod_list_all_doc, +"list_all() -> [qid]\n\ +\n\ +Return the list of IDs for all queues."); + +static PyObject * +queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", "obj", NULL}; + qidarg_converter_data qidarg; + PyObject *obj; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:put", kwlist, + qidarg_converter, &qidarg, &obj)) { + return NULL; + } + int64_t qid = qidarg.id; + + /* Queue up the object. */ + int err = queue_put(&_globals.queues, qid, obj); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_put_doc, +"put(qid, obj)\n\ +\n\ +Add the object's data to the queue."); + +static PyObject * +queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", "default", NULL}; + qidarg_converter_data qidarg; + PyObject *dflt = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O:get", kwlist, + qidarg_converter, &qidarg, &dflt)) { + return NULL; + } + int64_t qid = qidarg.id; + + PyObject *obj = NULL; + int err = queue_get(&_globals.queues, qid, &obj); + if (err == ERR_QUEUE_EMPTY && dflt != NULL) { + assert(obj == NULL); + obj = Py_NewRef(dflt); + } + else if (handle_queue_error(err, self, qid)) { + return NULL; + } + return obj; +} + +PyDoc_STRVAR(queuesmod_get_doc, +"get(qid, [default]) -> obj\n\ +\n\ +Return a new object from the data at the front of the queue.\n\ +\n\ +If there is nothing to receive then raise QueueEmpty, unless\n\ +a default value is provided. In that case return it."); + +static PyObject * +queuesmod_bind(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:bind", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + // XXX Check module state if bound already. + + int err = _queues_incref(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + + // XXX Update module state. + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_bind_doc, +"bind(qid)\n\ +\n\ +Take a reference to the identified queue.\n\ +The queue is not destroyed until there are no references left."); + +static PyObject * +queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) +{ + // Note that only the current interpreter is affected. + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:release", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + // XXX Check module state if bound already. + // XXX Update module state. + + _queues_decref(&_globals.queues, qid); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(queuesmod_release_doc, +"release(qid)\n\ +\n\ +Release a reference to the queue.\n\ +The queue is destroyed once there are no references left."); + +static PyObject * +queuesmod_get_maxsize(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_maxsize", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + Py_ssize_t maxsize = -1; + int err = queue_get_maxsize(&_globals.queues, qid, &maxsize); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + return PyLong_FromLongLong(maxsize); +} + +PyDoc_STRVAR(queuesmod_get_maxsize_doc, +"get_maxsize(qid)\n\ +\n\ +Return the maximum number of items in the queue."); + +static PyObject * +queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:is_full", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + int is_full = 0; + int err = queue_is_full(&_globals.queues, qid, &is_full); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + if (is_full) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(queuesmod_is_full_doc, +"is_full(qid)\n\ +\n\ +Return true if the queue has a maxsize and has reached it."); + +static PyObject * +queuesmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_count", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + Py_ssize_t count = -1; + int err = queue_get_count(&_globals.queues, qid, &count); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + assert(count >= 0); + return PyLong_FromSsize_t(count); +} + +PyDoc_STRVAR(queuesmod_get_count_doc, +"get_count(qid)\n\ +\n\ +Return the number of items in the queue."); + +static PyObject * +queuesmod__register_queue_type(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"queuetype", NULL}; + PyObject *queuetype; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:_register_queue_type", kwlist, + &queuetype)) { + return NULL; + } + if (!PyType_Check(queuetype)) { + PyErr_SetString(PyExc_TypeError, "expected a type for 'queuetype'"); + return NULL; + } + PyTypeObject *cls_queue = (PyTypeObject *)queuetype; + + if (set_external_queue_type(self, cls_queue) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +static PyMethodDef module_functions[] = { + {"create", _PyCFunction_CAST(queuesmod_create), + METH_VARARGS | METH_KEYWORDS, queuesmod_create_doc}, + {"destroy", _PyCFunction_CAST(queuesmod_destroy), + METH_VARARGS | METH_KEYWORDS, queuesmod_destroy_doc}, + {"list_all", queuesmod_list_all, + METH_NOARGS, queuesmod_list_all_doc}, + {"put", _PyCFunction_CAST(queuesmod_put), + METH_VARARGS | METH_KEYWORDS, queuesmod_put_doc}, + {"get", _PyCFunction_CAST(queuesmod_get), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_doc}, + {"bind", _PyCFunction_CAST(queuesmod_bind), + METH_VARARGS | METH_KEYWORDS, queuesmod_bind_doc}, + {"release", _PyCFunction_CAST(queuesmod_release), + METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, + {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, + {"is_full", _PyCFunction_CAST(queuesmod_is_full), + METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, + {"get_count", _PyCFunction_CAST(queuesmod_get_count), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_count_doc}, + {"_register_queue_type", _PyCFunction_CAST(queuesmod__register_queue_type), + METH_VARARGS | METH_KEYWORDS, NULL}, + + {NULL, NULL} /* sentinel */ +}; + + +/* initialization function */ + +PyDoc_STRVAR(module_doc, +"This module provides primitive operations to manage Python interpreters.\n\ +The 'interpreters' module provides a more convenient interface."); + +static int +module_exec(PyObject *mod) +{ + if (_globals_init() != 0) { + return -1; + } + + /* Add exception types */ + if (add_QueueError(mod) < 0) { + goto error; + } + + /* Make sure queues drop objects owned by this interpreter. */ + PyInterpreterState *interp = _get_current_interp(); + PyUnstable_AtExit(interp, clear_interpreter, (void *)interp); + + return 0; + +error: + _globals_fini(); + return -1; +} + +static struct PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + module_state *state = get_module_state(mod); + traverse_module_state(state, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + module_state *state = get_module_state(mod); + + if (state->queue_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->queue_type); + } + + // Now we clear the module state. + clear_module_state(state); + return 0; +} + +static void +module_free(void *mod) +{ + module_state *state = get_module_state(mod); + + // Now we clear the module state. + clear_module_state(state); + + _globals_fini(); +} + +static struct PyModuleDef moduledef = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = MODULE_NAME, + .m_doc = module_doc, + .m_size = sizeof(module_state), + .m_methods = module_functions, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = (freefunc)module_free, +}; + +PyMODINIT_FUNC +PyInit__xxinterpqueues(void) +{ + return PyModuleDef_Init(&moduledef); +} diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 02c2abed27ddfa..4e9e13457a9eb3 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -6,11 +6,14 @@ #endif #include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_crossinterp.h" // struct _xid -#include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() -#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() +#include "pycore_pyerrors.h" // _Py_excinfo #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "interpreteridobject.h" @@ -28,11 +31,260 @@ _get_current_interp(void) return PyInterpreterState_Get(); } +static int64_t +pylong_to_interpid(PyObject *idobj) +{ + assert(PyLong_CheckExact(idobj)); + + if (_PyLong_IsNegative((PyLongObject *)idobj)) { + PyErr_Format(PyExc_ValueError, + "interpreter ID must be a non-negative int, got %R", + idobj); + return -1; + } + + int overflow; + long long id = PyLong_AsLongLongAndOverflow(idobj, &overflow); + if (id == -1) { + if (!overflow) { + assert(PyErr_Occurred()); + return -1; + } + assert(!PyErr_Occurred()); + // For now, we don't worry about if LLONG_MAX < INT64_MAX. + goto bad_id; + } +#if LLONG_MAX > INT64_MAX + if (id > INT64_MAX) { + goto bad_id; + } +#endif + return (int64_t)id; + +bad_id: + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %O", idobj); + return -1; +} + +static int64_t +convert_interpid_obj(PyObject *arg) +{ + int64_t id = -1; + if (_PyIndex_Check(arg)) { + PyObject *idobj = PyNumber_Long(arg); + if (idobj == NULL) { + return -1; + } + id = pylong_to_interpid(idobj); + Py_DECREF(idobj); + if (id < 0) { + return -1; + } + } + else { + PyErr_Format(PyExc_TypeError, + "interpreter ID must be an int, got %.100s", + Py_TYPE(arg)->tp_name); + return -1; + } + return id; +} + +static PyInterpreterState * +look_up_interp(PyObject *arg) +{ + int64_t id = convert_interpid_obj(arg); + if (id < 0) { + return NULL; + } + return _PyInterpreterState_LookUpID(id); +} + + +static PyObject * +interpid_to_pylong(int64_t id) +{ + assert(id < LLONG_MAX); + return PyLong_FromLongLong(id); +} + +static PyObject * +get_interpid_obj(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IDInitref(interp) != 0) { + return NULL; + }; + int64_t id = PyInterpreterState_GetID(interp); + if (id < 0) { + return NULL; + } + return interpid_to_pylong(id); +} + +static PyObject * +_get_current_module(void) +{ + PyObject *name = PyUnicode_FromString(MODULE_NAME); + if (name == NULL) { + return NULL; + } + PyObject *mod = PyImport_GetModule(name); + Py_DECREF(name); + if (mod == NULL) { + return NULL; + } + assert(mod != Py_None); + return mod; +} + + +/* Cross-interpreter Buffer Views *******************************************/ + +// XXX Release when the original interpreter is destroyed. + +typedef struct { + PyObject_HEAD + Py_buffer *view; + int64_t interpid; +} XIBufferViewObject; + +static PyObject * +xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) +{ + assert(data->data != NULL); + assert(data->obj == NULL); + assert(data->interpid >= 0); + XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); + if (self == NULL) { + return NULL; + } + PyObject_Init((PyObject *)self, cls); + self->view = (Py_buffer *)data->data; + self->interpid = data->interpid; + return (PyObject *)self; +} + +static void +xibufferview_dealloc(XIBufferViewObject *self) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interpid); + /* If the interpreter is no longer alive then we have problems, + since other objects may be using the buffer still. */ + assert(interp != NULL); + + if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { + // XXX Emit a warning? + PyErr_Clear(); + } + + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + /* "Instances of heap-allocated types hold a reference to their type." + * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol + * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse + */ + // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, + // like we do for _abc._abc_data? + Py_DECREF(tp); +} + +static int +xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags) +{ + /* Only PyMemoryView_FromObject() should ever call this, + via _memoryview_from_xid() below. */ + *view = *self->view; + view->obj = (PyObject *)self; + // XXX Should we leave it alone? + view->internal = NULL; + return 0; +} + +static PyType_Slot XIBufferViewType_slots[] = { + {Py_tp_dealloc, (destructor)xibufferview_dealloc}, + {Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf}, + // We don't bother with Py_bf_releasebuffer since we don't need it. + {0, NULL}, +}; + +static PyType_Spec XIBufferViewType_spec = { + .name = MODULE_NAME ".CrossInterpreterBufferView", + .basicsize = sizeof(XIBufferViewObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + .slots = XIBufferViewType_slots, +}; + + +static PyTypeObject * _get_current_xibufferview_type(void); + +static PyObject * +_memoryview_from_xid(_PyCrossInterpreterData *data) +{ + PyTypeObject *cls = _get_current_xibufferview_type(); + if (cls == NULL) { + return NULL; + } + PyObject *obj = xibufferview_from_xid(cls, data); + if (obj == NULL) { + return NULL; + } + return PyMemoryView_FromObject(obj); +} + +static int +_memoryview_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); + if (view == NULL) { + return -1; + } + if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) { + PyMem_RawFree(view); + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, + _memoryview_from_xid); + return 0; +} + +static int +register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) +{ + // XIBufferView + assert(*p_state == NULL); + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &XIBufferViewType_spec, NULL); + if (cls == NULL) { + return -1; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); + return -1; + } + *p_state = cls; + + // Register XID for the builtin memoryview type. + if (_PyCrossInterpreterData_RegisterClass( + &PyMemoryView_Type, _memoryview_shared) < 0) { + return -1; + } + // We don't ever bother un-registering memoryview. + + return 0; +} + + /* module state *************************************************************/ typedef struct { int _notused; + + /* heap types */ + PyTypeObject *XIBufferViewType; } module_state; static inline module_state * @@ -44,19 +296,51 @@ get_module_state(PyObject *mod) return state; } +static module_state * +_get_current_module_state(void) +{ + PyObject *mod = _get_current_module(); + if (mod == NULL) { + // XXX import it? + PyErr_SetString(PyExc_RuntimeError, + MODULE_NAME " module not imported yet"); + return NULL; + } + module_state *state = get_module_state(mod); + Py_DECREF(mod); + return state; +} + static int traverse_module_state(module_state *state, visitproc visit, void *arg) { + /* heap types */ + Py_VISIT(state->XIBufferViewType); + return 0; } static int clear_module_state(module_state *state) { + /* heap types */ + Py_CLEAR(state->XIBufferViewType); + return 0; } +static PyTypeObject * +_get_current_xibufferview_type(void) +{ + module_state *state = _get_current_module_state(); + if (state == NULL) { + return NULL; + } + return state->XIBufferViewType; +} + + /* Python code **************************************************************/ static const char * @@ -254,7 +538,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) assert(tstate != NULL); PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = PyInterpreterState_GetIDObject(interp); + PyObject *idobj = get_interpid_obj(interp); if (idobj == NULL) { // XXX Possible GILState issues? save_tstate = PyThreadState_Swap(tstate); @@ -263,7 +547,9 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } + PyThreadState_Swap(tstate); PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); PyThreadState_Delete(tstate); _PyInterpreterState_RequireIDRef(interp, 1); @@ -273,7 +559,9 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) PyDoc_STRVAR(create_doc, "create() -> ID\n\ \n\ -Create a new interpreter and return a unique generated ID."); +Create a new interpreter and return a unique generated ID.\n\ +\n\ +The caller is responsible for destroying the interpreter before exiting."); static PyObject * @@ -288,7 +576,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) } // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + PyInterpreterState *interp = look_up_interp(id); if (interp == NULL) { return NULL; } @@ -345,7 +633,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) interp = PyInterpreterState_Head(); while (interp != NULL) { - id = PyInterpreterState_GetIDObject(interp); + id = get_interpid_obj(interp); if (id == NULL) { Py_DECREF(ids); return NULL; @@ -377,7 +665,7 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) if (interp == NULL) { return NULL; } - return PyInterpreterState_GetIDObject(interp); + return get_interpid_obj(interp); } PyDoc_STRVAR(get_current_doc, @@ -391,7 +679,7 @@ interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) { // Currently, 0 is always the main interpreter. int64_t id = 0; - return PyInterpreterID_New(id); + return PyLong_FromLongLong(id); } PyDoc_STRVAR(get_main_doc, @@ -399,6 +687,60 @@ PyDoc_STRVAR(get_main_doc, \n\ Return the ID of main interpreter."); +static PyObject * +interp_set___main___attrs(PyObject *self, PyObject *args) +{ + PyObject *id, *updates; + if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME ".set___main___attrs", + &id, &updates)) + { + return NULL; + } + + // Look up the interpreter. + PyInterpreterState *interp = PyInterpreterID_LookUp(id); + if (interp == NULL) { + return NULL; + } + + // Check the updates. + if (updates != Py_None) { + Py_ssize_t size = PyObject_Size(updates); + if (size < 0) { + return NULL; + } + if (size == 0) { + PyErr_SetString(PyExc_ValueError, + "arg 2 must be a non-empty mapping"); + return NULL; + } + } + + _PyXI_session session = {0}; + + // Prep and switch interpreters, including apply the updates. + if (_PyXI_Enter(&session, interp, updates) < 0) { + if (!PyErr_Occurred()) { + _PyXI_ApplyCapturedException(&session); + assert(PyErr_Occurred()); + } + else { + assert(!_PyXI_HasCapturedException(&session)); + } + return NULL; + } + + // Clean up and switch back. + _PyXI_Exit(&session); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(set___main___attrs_doc, +"set___main___attrs(id, ns)\n\ +\n\ +Bind the given attributes in the interpreter's __main__ module."); + static PyUnicodeObject * convert_script_arg(PyObject *arg, const char *fname, const char *displayname, const char *expected) @@ -481,7 +823,7 @@ _interp_exec(PyObject *self, PyObject **p_excinfo) { // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id_arg); + PyInterpreterState *interp = look_up_interp(id_arg); if (interp == NULL) { return -1; } @@ -667,7 +1009,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + PyInterpreterState *interp = look_up_interp(id); if (interp == NULL) { return NULL; } @@ -683,6 +1025,49 @@ PyDoc_STRVAR(is_running_doc, Return whether or not the identified interpreter is running."); +static PyObject * +interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *id; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:_incref", kwlist, &id)) { + return NULL; + } + + PyInterpreterState *interp = look_up_interp(id); + if (interp == NULL) { + return NULL; + } + if (_PyInterpreterState_IDInitref(interp) < 0) { + return NULL; + } + _PyInterpreterState_IDIncref(interp); + + Py_RETURN_NONE; +} + + +static PyObject * +interp_decref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *id; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:_incref", kwlist, &id)) { + return NULL; + } + + PyInterpreterState *interp = look_up_interp(id); + if (interp == NULL) { + return NULL; + } + _PyInterpreterState_IDDecref(interp); + + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, @@ -704,9 +1089,16 @@ static PyMethodDef module_functions[] = { {"run_func", _PyCFunction_CAST(interp_run_func), METH_VARARGS | METH_KEYWORDS, run_func_doc}, + {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), + METH_VARARGS, set___main___attrs_doc}, {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + {"_incref", _PyCFunction_CAST(interp_incref), + METH_VARARGS | METH_KEYWORDS, NULL}, + {"_decref", _PyCFunction_CAST(interp_decref), + METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL} /* sentinel */ }; @@ -720,8 +1112,17 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { - // PyInterpreterID - if (PyModule_AddType(mod, &PyInterpreterID_Type) < 0) { + module_state *state = get_module_state(mod); + + // exceptions + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { + goto error; + } + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { + goto error; + } + + if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) { goto error; } diff --git a/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict b/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict new file mode 100644 index 00000000000000..c6a44d946284ef --- /dev/null +++ b/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict @@ -0,0 +1,165 @@ +# bits of syntax +"( " +") " +"[ " +"] " +": " +", " +"; " +"{ " +"} " + +# operators +"+ " +"- " +"* " +"** " +"/ " +"// " +"| " +"& " +"< " +"> " +"= " +". " +"% " +"` " +"^ " +"~ " +"@ " +"== " +"!= " +"<> " +"<< " +"<= " +">= " +">> " +"+= " +"-= " +"*= " +"** " +"/= " +"//= " +"|= " +"%= " +"&= " +"^= " +"<<= " +">>= " +"**= " +":= " +"@= " + +# whitespace +" " +":\\n " + +# type signatures and functions +"-> " +": List[int]" +": Dict[int, str]" + +"# type:" +"# type: List[int]" +"# type: Dict[int, str]" + +", *" +", /" +", *args" +", **kwargs" +", x=42" + + +# literals +"0x0a" +"0b0000" +"42" +"0o70" +"42j" +"42.01" +"-5" +"+42e-3" +"0_0_0" +"1e1_0" +".1_4" + +"{}" + +# variable names +"x" +"y" + +# strings +"r'x'" + +"b'x'" + +"rb\"x\"" + +"br\"x\"" + +"f'{x + 5}'" +"f\"{x + 5}\"" + +"'''" +"\"\"\"" + +"\\u" +"\\x" + +# keywords +"def " +"del " +"pass " +"break " +"continue " +"return " +"raise " +"from " +"import " +".. " +"... " +"__future__ " +"as " +"global " +"nonlocal " +"assert " +"print " +"if " +"elif " +"else: " +"while " +"try: " +"except " +"finally: " +"with " +"lambda " +"or " +"and " +"not " +"None " +"__peg_parser__" +"True " +"False " +"yield " +"async " +"await " +"for " +"in " +"is " +"class " + +# shebangs and encodings +"#!" +"# coding:" +"# coding=" +"# coding: latin-1" +"# coding=latin-1" +"# coding: utf-8" +"# coding=utf-8" +"# coding: ascii" +"# coding=ascii" +"# coding: cp860" +"# coding=cp860" +"# coding: gbk" +"# coding=gbk" diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py new file mode 100644 index 00000000000000..c43994dda29eed --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +def test() -> None: + x: list[int] = [] + x: dict[int, str] = {} + x: set[bytes] = {} + print(5 + 42 * 3, x) diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py new file mode 100644 index 00000000000000..7be326e95be0eb --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py @@ -0,0 +1,5 @@ +class Foo(metaclass=42): + __slots__ = ['x'] + pass + +foo = Foo() diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py new file mode 100644 index 00000000000000..9bc3a45ebe75da --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py @@ -0,0 +1,6 @@ +def evens(): + i = 0 + while True: + i += 1 + if i % 2 == 0: + yield i diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py new file mode 100644 index 00000000000000..490de90fb97b39 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py @@ -0,0 +1,3 @@ +async def hello(name: str): + await name + print(name) diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py new file mode 100644 index 00000000000000..4cfcfe590ebc95 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py @@ -0,0 +1,7 @@ +try: + eval('importer exporter... really long matches') +except SyntaxError: + print("nothing to see here") +finally: + print("all done here") + raise diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py new file mode 100644 index 00000000000000..d8e59ade503a8c --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py @@ -0,0 +1,8 @@ +"""Some module docstring""" +import sys + +def main(): + print("Hello world!", file=sys.stderr) + +if __name__ == '__main__': + main() diff --git a/Modules/_xxtestfuzz/fuzz_tests.txt b/Modules/_xxtestfuzz/fuzz_tests.txt index 40aa22110e7d27..ea6f982eefc9da 100644 --- a/Modules/_xxtestfuzz/fuzz_tests.txt +++ b/Modules/_xxtestfuzz/fuzz_tests.txt @@ -8,3 +8,4 @@ fuzz_csv_reader fuzz_struct_unpack fuzz_ast_literal_eval fuzz_elementtree_parsewhole +fuzz_pycompile diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index 77d29ce773a04b..e133b4d3c44480 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -501,6 +501,63 @@ static int fuzz_elementtree_parsewhole(const char* data, size_t size) { return 0; } +#define MAX_PYCOMPILE_TEST_SIZE 16384 +static char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; + +static const int start_vals[] = {Py_eval_input, Py_single_input, Py_file_input}; +const size_t NUM_START_VALS = sizeof(start_vals) / sizeof(start_vals[0]); + +static const int optimize_vals[] = {-1, 0, 1, 2}; +const size_t NUM_OPTIMIZE_VALS = sizeof(optimize_vals) / sizeof(optimize_vals[0]); + +/* Fuzz `PyCompileStringExFlags` using a variety of input parameters. + * That function is essentially behind the `compile` builtin */ +static int fuzz_pycompile(const char* data, size_t size) { + // Ignore overly-large inputs, and account for a NUL terminator + if (size > MAX_PYCOMPILE_TEST_SIZE - 1) { + return 0; + } + + // Need 2 bytes for parameter selection + if (size < 2) { + return 0; + } + + // Use first byte to determine element of `start_vals` to use + unsigned char start_idx = (unsigned char) data[0]; + int start = start_vals[start_idx % NUM_START_VALS]; + + // Use second byte to determine element of `optimize_vals` to use + unsigned char optimize_idx = (unsigned char) data[1]; + int optimize = optimize_vals[optimize_idx % NUM_OPTIMIZE_VALS]; + + // Create a NUL-terminated C string from the remaining input + memcpy(pycompile_scratch, data + 2, size - 2); + // Put a NUL terminator just after the copied data. (Space was reserved already.) + pycompile_scratch[size - 2] = '\0'; + + // XXX: instead of always using NULL for the `flags` value to + // `Py_CompileStringExFlags`, there are many flags that conditionally + // change parser behavior: + // + // #define PyCF_TYPE_COMMENTS 0x1000 + // #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 + // #define PyCF_ONLY_AST 0x0400 + // + // It would be good to test various combinations of these, too. + PyCompilerFlags *flags = NULL; + + PyObject *result = Py_CompileStringExFlags(pycompile_scratch, "", start, flags, optimize); + if (result == NULL) { + /* compilation failed, most likely from a syntax error */ + PyErr_Clear(); + } else { + Py_DECREF(result); + } + + return 0; +} + /* Run fuzzer and abort on failure. */ static int _run_fuzz(const uint8_t *data, size_t size, int(*fuzzer)(const char* , size_t)) { int rv = fuzzer((const char*) data, size); @@ -642,6 +699,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { } rv |= _run_fuzz(data, size, fuzz_elementtree_parsewhole); +#endif +#if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_pycompile) + rv |= _run_fuzz(data, size, fuzz_pycompile); #endif return rv; } diff --git a/Modules/binascii.c b/Modules/binascii.c index 17970aa5e9456c..86493241a1fb7e 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -770,12 +770,20 @@ binascii_crc32_impl(PyObject *module, Py_buffer *data, unsigned int crc) Py_BEGIN_ALLOW_THREADS /* Avoid truncation of length for very large buffers. crc32() takes - length as an unsigned int, which may be narrower than Py_ssize_t. */ - while ((size_t)len > UINT_MAX) { - crc = crc32(crc, buf, UINT_MAX); - buf += (size_t) UINT_MAX; - len -= (size_t) UINT_MAX; + length as an unsigned int, which may be narrower than Py_ssize_t. + We further limit size due to bugs in Apple's macOS zlib. + See https://github.com/python/cpython/issues/105967 + */ +#define ZLIB_CRC_CHUNK_SIZE 0x40000000 +#if ZLIB_CRC_CHUNK_SIZE > INT_MAX +# error "unsupported less than 32-bit platform?" +#endif + while ((size_t)len > ZLIB_CRC_CHUNK_SIZE) { + crc = crc32(crc, buf, ZLIB_CRC_CHUNK_SIZE); + buf += (size_t) ZLIB_CRC_CHUNK_SIZE; + len -= (size_t) ZLIB_CRC_CHUNK_SIZE; } +#undef ZLIB_CRC_CHUNK_SIZE crc = crc32(crc, buf, (unsigned int)len); Py_END_ALLOW_THREADS } else { diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 757e49e23cacfb..6193acac67e7ac 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_random_Random_random__doc__, @@ -19,7 +20,13 @@ _random_Random_random_impl(RandomObject *self); static PyObject * _random_Random_random(RandomObject *self, PyObject *Py_UNUSED(ignored)) { - return _random_Random_random_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _random_Random_random_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_random_Random_seed__doc__, @@ -51,7 +58,9 @@ _random_Random_seed(RandomObject *self, PyObject *const *args, Py_ssize_t nargs) } n = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _random_Random_seed_impl(self, n); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -72,7 +81,13 @@ _random_Random_getstate_impl(RandomObject *self); static PyObject * _random_Random_getstate(RandomObject *self, PyObject *Py_UNUSED(ignored)) { - return _random_Random_getstate_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _random_Random_getstate_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_random_Random_setstate__doc__, @@ -84,6 +99,21 @@ PyDoc_STRVAR(_random_Random_setstate__doc__, #define _RANDOM_RANDOM_SETSTATE_METHODDEF \ {"setstate", (PyCFunction)_random_Random_setstate, METH_O, _random_Random_setstate__doc__}, +static PyObject * +_random_Random_setstate_impl(RandomObject *self, PyObject *state); + +static PyObject * +_random_Random_setstate(RandomObject *self, PyObject *state) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _random_Random_setstate_impl(self, state); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_random_Random_getrandbits__doc__, "getrandbits($self, k, /)\n" "--\n" @@ -106,9 +136,11 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg) if (k == -1 && PyErr_Occurred()) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _random_Random_getrandbits_impl(self, k); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=5c800a28c2d7b9d1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bf49ece1d341b1b6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index fa2c5e0e922387..3ec479943a83d4 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -10,7 +10,7 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(batched_new__doc__, -"batched(iterable, n)\n" +"batched(iterable, n, *, strict=False)\n" "--\n" "\n" "Batch data into tuples of length n. The last batch may be shorter than n.\n" @@ -25,10 +25,14 @@ PyDoc_STRVAR(batched_new__doc__, " ...\n" " (\'A\', \'B\', \'C\')\n" " (\'D\', \'E\', \'F\')\n" -" (\'G\',)"); +" (\'G\',)\n" +"\n" +"If \"strict\" is True, raises a ValueError if the final batch is shorter\n" +"than n."); static PyObject * -batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n); +batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n, + int strict); static PyObject * batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -36,14 +40,14 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(n), }, + .ob_item = { &_Py_ID(iterable), &_Py_ID(n), &_Py_ID(strict), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -52,18 +56,20 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"iterable", "n", NULL}; + static const char * const _keywords[] = {"iterable", "n", "strict", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "batched", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; PyObject *iterable; Py_ssize_t n; + int strict = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); if (!fastargs) { @@ -82,7 +88,15 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } n = ival; } - return_value = batched_new_impl(type, iterable, n); + if (!noptargs) { + goto skip_optional_kwonly; + } + strict = PyObject_IsTrue(fastargs[2]); + if (strict < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = batched_new_impl(type, iterable, n, strict); exit: return return_value; @@ -914,4 +928,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=782fe7e30733779b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6a515f765da86b5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 9c54935bafa617..ba3e1cfa8dbc21 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -493,7 +493,8 @@ os_fchdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #endif /* defined(HAVE_FCHDIR) */ PyDoc_STRVAR(os_chmod__doc__, -"chmod($module, /, path, mode, *, dir_fd=None, follow_symlinks=True)\n" +"chmod($module, /, path, mode, *, dir_fd=None,\n" +" follow_symlinks=(os.name != \'nt\'))\n" "--\n" "\n" "Change the access permissions of a file.\n" @@ -562,7 +563,7 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw path_t path = PATH_T_INITIALIZE("chmod", "path", 0, PATH_HAVE_FCHMOD); int mode; int dir_fd = DEFAULT_DIR_FD; - int follow_symlinks = 1; + int follow_symlinks = CHMOD_DEFAULT_FOLLOW_SYMLINKS; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); if (!args) { @@ -600,7 +601,7 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw return return_value; } -#if defined(HAVE_FCHMOD) +#if (defined(HAVE_FCHMOD) || defined(MS_WINDOWS)) PyDoc_STRVAR(os_fchmod__doc__, "fchmod($module, /, fd, mode)\n" @@ -675,9 +676,9 @@ os_fchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } -#endif /* defined(HAVE_FCHMOD) */ +#endif /* (defined(HAVE_FCHMOD) || defined(MS_WINDOWS)) */ -#if defined(HAVE_LCHMOD) +#if (defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) PyDoc_STRVAR(os_lchmod__doc__, "lchmod($module, /, path, mode)\n" @@ -747,7 +748,7 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } -#endif /* defined(HAVE_LCHMOD) */ +#endif /* (defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) */ #if defined(HAVE_CHFLAGS) @@ -5466,7 +5467,7 @@ os_wait4(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #endif /* defined(HAVE_WAIT4) */ -#if (defined(HAVE_WAITID) && !defined(__APPLE__)) +#if defined(HAVE_WAITID) PyDoc_STRVAR(os_waitid__doc__, "waitid($module, idtype, id, options, /)\n" @@ -5509,7 +5510,7 @@ os_waitid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* (defined(HAVE_WAITID) && !defined(__APPLE__)) */ +#endif /* defined(HAVE_WAITID) */ #if defined(HAVE_WAITPID) @@ -5997,8 +5998,6 @@ os_symlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #endif /* defined(HAVE_SYMLINK) */ -#if defined(HAVE_TIMES) - PyDoc_STRVAR(os_times__doc__, "times($module, /)\n" "--\n" @@ -6021,8 +6020,6 @@ os_times(PyObject *module, PyObject *Py_UNUSED(ignored)) return os_times_impl(module); } -#endif /* defined(HAVE_TIMES) */ - #if defined(HAVE_TIMERFD_CREATE) PyDoc_STRVAR(os_timerfd_create__doc__, @@ -11760,6 +11757,28 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #endif /* (defined(WIFEXITED) || defined(MS_WINDOWS)) */ +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os__supports_virtual_terminal__doc__, +"_supports_virtual_terminal($module, /)\n" +"--\n" +"\n" +"Checks if virtual terminal is supported in windows"); + +#define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF \ + {"_supports_virtual_terminal", (PyCFunction)os__supports_virtual_terminal, METH_NOARGS, os__supports_virtual_terminal__doc__}, + +static PyObject * +os__supports_virtual_terminal_impl(PyObject *module); + +static PyObject * +os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__supports_virtual_terminal_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -12116,10 +12135,6 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_SYMLINK_METHODDEF #endif /* !defined(OS_SYMLINK_METHODDEF) */ -#ifndef OS_TIMES_METHODDEF - #define OS_TIMES_METHODDEF -#endif /* !defined(OS_TIMES_METHODDEF) */ - #ifndef OS_TIMERFD_CREATE_METHODDEF #define OS_TIMERFD_CREATE_METHODDEF #endif /* !defined(OS_TIMERFD_CREATE_METHODDEF) */ @@ -12403,4 +12418,8 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=0f216bf44ea358f9 input=a9049054013a1b77]*/ + +#ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF + #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF +#endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ +/*[clinic end generated code: output=18c128534c355d84 input=a9049054013a1b77]*/ diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 8287edbfb47f6c..1100e9f6094352 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,8 +1,6 @@ /* Errno module */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 568e02a4210a2b..2d1f381e622226 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -74,6 +74,20 @@ module gc #define AS_GC(op) _Py_AS_GC(op) #define FROM_GC(gc) _Py_FROM_GC(gc) +// Automatically choose the generation that needs collecting. +#define GENERATION_AUTO (-1) + +typedef enum { + // GC was triggered by heap allocation + _Py_GC_REASON_HEAP, + + // GC was called during shutdown + _Py_GC_REASON_SHUTDOWN, + + // GC was called by gc.collect() or PyGC_Collect() + _Py_GC_REASON_MANUAL +} _PyGC_Reason; + static inline int gc_is_collecting(PyGC_Head *g) @@ -491,15 +505,16 @@ subtract_refs(PyGC_Head *containers) PyObject *op = FROM_GC(gc); traverse = Py_TYPE(op)->tp_traverse; (void) traverse(op, - (visitproc)visit_decref, + visit_decref, op); } } /* A traversal callback for move_unreachable. */ static int -visit_reachable(PyObject *op, PyGC_Head *reachable) +visit_reachable(PyObject *op, void *arg) { + PyGC_Head *reachable = arg; OBJECT_STAT_INC(object_visits); if (!_PyObject_IS_GC(op)) { return 0; @@ -603,7 +618,7 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) // NOTE: visit_reachable may change gc->_gc_next when // young->_gc_prev == gc. Don't do gc = GC_NEXT(gc) before! (void) traverse(op, - (visitproc)visit_reachable, + visit_reachable, (void *)young); // relink gc_prev to prev element. _PyGCHead_SET_PREV(gc, prev); @@ -726,8 +741,9 @@ clear_unreachable_mask(PyGC_Head *unreachable) /* A traversal callback for move_legacy_finalizer_reachable. */ static int -visit_move(PyObject *op, PyGC_Head *tolist) +visit_move(PyObject *op, void *arg) { + PyGC_Head *tolist = arg; OBJECT_STAT_INC(object_visits); if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); @@ -751,7 +767,7 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) /* Note that the finalizers list may grow during this. */ traverse = Py_TYPE(FROM_GC(gc))->tp_traverse; (void) traverse(FROM_GC(gc), - (visitproc)visit_move, + visit_move, (void *)finalizers); } } @@ -1192,19 +1208,122 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, gc_list_merge(resurrected, old_generation); } + +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +invoke_gc_callback(PyThreadState *tstate, const char *phase, + int generation, Py_ssize_t collected, + Py_ssize_t uncollectable) +{ + assert(!_PyErr_Occurred(tstate)); + + /* we may get called very early */ + GCState *gcstate = &tstate->interp->gc; + if (gcstate->callbacks == NULL) { + return; + } + + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", collected, + "uncollectable", uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } + + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; icallbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_WriteUnraisable(cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); + } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!_PyErr_Occurred(tstate)); +} + + +/* Find the oldest generation (highest numbered) where the count + * exceeds the threshold. Objects in the that generation and + * generations younger than it will be collected. */ +static int +gc_select_generation(GCState *gcstate) +{ + for (int i = NUM_GENERATIONS-1; i >= 0; i--) { + if (gcstate->generations[i].count > gcstate->generations[i].threshold) { + /* Avoid quadratic performance degradation in number + of tracked objects (see also issue #4074): + + To limit the cost of garbage collection, there are two strategies; + - make each collection faster, e.g. by scanning fewer objects + - do less collections + This heuristic is about the latter strategy. + + In addition to the various configurable thresholds, we only trigger a + full collection if the ratio + + long_lived_pending / long_lived_total + + is above a given value (hardwired to 25%). + + The reason is that, while "non-full" collections (i.e., collections of + the young and middle generations) will always examine roughly the same + number of objects -- determined by the aforementioned thresholds --, + the cost of a full collection is proportional to the total number of + long-lived objects, which is virtually unbounded. + + Indeed, it has been remarked that doing a full collection every + of object creations entails a dramatic performance + degradation in workloads which consist in creating and storing lots of + long-lived objects (e.g. building a large list of GC-tracked objects would + show quadratic performance, instead of linear as expected: see issue #4074). + + Using the above ratio, instead, yields amortized linear performance in + the total number of objects (the effect of which can be summarized + thusly: "each full garbage collection is more and more costly as the + number of objects grows, but we do fewer and fewer of them"). + + This heuristic was suggested by Martin von Löwis on python-dev in + June 2008. His original analysis and proposal can be found at: + http://mail.python.org/pipermail/python-dev/2008-June/080579.html + */ + if (i == NUM_GENERATIONS - 1 + && gcstate->long_lived_pending < gcstate->long_lived_total / 4) + continue; + return i; + } + } + return -1; +} + + /* This is the main function. Read this to understand how the * collection process works. */ static Py_ssize_t -gc_collect_main(PyThreadState *tstate, int generation, - Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, - int nofail) +gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) { - GC_STAT_ADD(generation, collections, 1); -#ifdef Py_STATS - if (_Py_stats) { - _Py_stats->object_stats.object_visits = 0; - } -#endif int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ @@ -1221,6 +1340,36 @@ gc_collect_main(PyThreadState *tstate, int generation, assert(gcstate->garbage != NULL); assert(!_PyErr_Occurred(tstate)); + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. + return 0; + } + + if (generation == GENERATION_AUTO) { + // Select the oldest generation that needs collecting. We will collect + // objects from that generation and all generations younger than it. + generation = gc_select_generation(gcstate); + if (generation < 0) { + // No generation needs to be collected. + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; + } + } + + assert(generation >= 0 && generation < NUM_GENERATIONS); + +#ifdef Py_STATS + if (_Py_stats) { + _Py_stats->object_stats.object_visits = 0; + } +#endif + GC_STAT_ADD(generation, collections, 1); + + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "start", generation, 0, 0); + } + if (gcstate->debug & DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); @@ -1340,7 +1489,7 @@ gc_collect_main(PyThreadState *tstate, int generation, } if (_PyErr_Occurred(tstate)) { - if (nofail) { + if (reason == _Py_GC_REASON_SHUTDOWN) { _PyErr_Clear(tstate); } else { @@ -1349,13 +1498,6 @@ gc_collect_main(PyThreadState *tstate, int generation, } /* Update stats */ - if (n_collected) { - *n_collected = m; - } - if (n_uncollectable) { - *n_uncollectable = n; - } - struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; stats->collections++; stats->collected += m; @@ -1374,134 +1516,13 @@ gc_collect_main(PyThreadState *tstate, int generation, PyDTrace_GC_DONE(n + m); } - assert(!_PyErr_Occurred(tstate)); - return n + m; -} - -/* Invoke progress callbacks to notify clients that garbage collection - * is starting or stopping - */ -static void -invoke_gc_callback(PyThreadState *tstate, const char *phase, - int generation, Py_ssize_t collected, - Py_ssize_t uncollectable) -{ - assert(!_PyErr_Occurred(tstate)); - - /* we may get called very early */ - GCState *gcstate = &tstate->interp->gc; - if (gcstate->callbacks == NULL) { - return; - } - - /* The local variable cannot be rebound, check it for sanity */ - assert(PyList_CheckExact(gcstate->callbacks)); - PyObject *info = NULL; - if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsn}", - "generation", generation, - "collected", collected, - "uncollectable", uncollectable); - if (info == NULL) { - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; - } + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "stop", generation, m, n); } - PyObject *phase_obj = PyUnicode_FromString(phase); - if (phase_obj == NULL) { - Py_XDECREF(info); - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; - } - - PyObject *stack[] = {phase_obj, info}; - for (Py_ssize_t i=0; icallbacks); i++) { - PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); - Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_Vectorcall(cb, stack, 2, NULL); - if (r == NULL) { - PyErr_WriteUnraisable(cb); - } - else { - Py_DECREF(r); - } - Py_DECREF(cb); - } - Py_DECREF(phase_obj); - Py_XDECREF(info); - assert(!_PyErr_Occurred(tstate)); -} - -/* Perform garbage collection of a generation and invoke - * progress callbacks. - */ -static Py_ssize_t -gc_collect_with_callback(PyThreadState *tstate, int generation) -{ assert(!_PyErr_Occurred(tstate)); - Py_ssize_t result, collected, uncollectable; - invoke_gc_callback(tstate, "start", generation, 0, 0); - result = gc_collect_main(tstate, generation, &collected, &uncollectable, 0); - invoke_gc_callback(tstate, "stop", generation, collected, uncollectable); - assert(!_PyErr_Occurred(tstate)); - return result; -} - -static Py_ssize_t -gc_collect_generations(PyThreadState *tstate) -{ - GCState *gcstate = &tstate->interp->gc; - /* Find the oldest generation (highest numbered) where the count - * exceeds the threshold. Objects in the that generation and - * generations younger than it will be collected. */ - Py_ssize_t n = 0; - for (int i = NUM_GENERATIONS-1; i >= 0; i--) { - if (gcstate->generations[i].count > gcstate->generations[i].threshold) { - /* Avoid quadratic performance degradation in number - of tracked objects (see also issue #4074): - - To limit the cost of garbage collection, there are two strategies; - - make each collection faster, e.g. by scanning fewer objects - - do less collections - This heuristic is about the latter strategy. - - In addition to the various configurable thresholds, we only trigger a - full collection if the ratio - - long_lived_pending / long_lived_total - - is above a given value (hardwired to 25%). - - The reason is that, while "non-full" collections (i.e., collections of - the young and middle generations) will always examine roughly the same - number of objects -- determined by the aforementioned thresholds --, - the cost of a full collection is proportional to the total number of - long-lived objects, which is virtually unbounded. - - Indeed, it has been remarked that doing a full collection every - of object creations entails a dramatic performance - degradation in workloads which consist in creating and storing lots of - long-lived objects (e.g. building a large list of GC-tracked objects would - show quadratic performance, instead of linear as expected: see issue #4074). - - Using the above ratio, instead, yields amortized linear performance in - the total number of objects (the effect of which can be summarized - thusly: "each full garbage collection is more and more costly as the - number of objects grows, but we do fewer and fewer of them"). - - This heuristic was suggested by Martin von Löwis on python-dev in - June 2008. His original analysis and proposal can be found at: - http://mail.python.org/pipermail/python-dev/2008-June/080579.html - */ - if (i == NUM_GENERATIONS - 1 - && gcstate->long_lived_pending < gcstate->long_lived_total / 4) - continue; - n = gc_collect_with_callback(tstate, i); - break; - } - } - return n; + _Py_atomic_store_int(&gcstate->collecting, 0); + return n + m; } #include "clinic/gcmodule.c.h" @@ -1572,18 +1593,7 @@ gc_collect_impl(PyObject *module, int generation) return -1; } - GCState *gcstate = &tstate->interp->gc; - Py_ssize_t n; - if (gcstate->collecting) { - /* already collecting, don't do anything */ - n = 0; - } - else { - gcstate->collecting = 1; - n = gc_collect_with_callback(tstate, generation); - gcstate->collecting = 0; - } - return n; + return gc_collect_main(tstate, generation, _Py_GC_REASON_MANUAL); } /*[clinic input] @@ -1684,8 +1694,9 @@ gc_get_count_impl(PyObject *module) } static int -referrersvisit(PyObject* obj, PyObject *objs) +referrersvisit(PyObject* obj, void *arg) { + PyObject *objs = arg; Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(objs); i++) if (PyTuple_GET_ITEM(objs, i) == obj) @@ -1704,7 +1715,7 @@ gc_referrers_for(PyObject *objs, PyGC_Head *list, PyObject *resultlist) traverse = Py_TYPE(obj)->tp_traverse; if (obj == objs || obj == resultlist) continue; - if (traverse(obj, (visitproc)referrersvisit, objs)) { + if (traverse(obj, referrersvisit, objs)) { if (PyList_Append(resultlist, obj) < 0) return 0; /* error */ } @@ -1740,8 +1751,9 @@ gc_get_referrers(PyObject *self, PyObject *args) /* Append obj to list; return true if error (out of memory), false if OK. */ static int -referentsvisit(PyObject *obj, PyObject *list) +referentsvisit(PyObject *obj, void *arg) { + PyObject *list = arg; return PyList_Append(list, obj) < 0; } @@ -1770,7 +1782,7 @@ gc_get_referents(PyObject *self, PyObject *args) traverse = Py_TYPE(obj)->tp_traverse; if (! traverse) continue; - if (traverse(obj, (visitproc)referentsvisit, result)) { + if (traverse(obj, referentsvisit, result)) { Py_DECREF(result); return NULL; } @@ -2120,17 +2132,9 @@ PyGC_Collect(void) } Py_ssize_t n; - if (gcstate->collecting) { - /* already collecting, don't do anything */ - n = 0; - } - else { - gcstate->collecting = 1; - PyObject *exc = _PyErr_GetRaisedException(tstate); - n = gc_collect_with_callback(tstate, NUM_GENERATIONS - 1); - _PyErr_SetRaisedException(tstate, exc); - gcstate->collecting = 0; - } + PyObject *exc = _PyErr_GetRaisedException(tstate); + n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); + _PyErr_SetRaisedException(tstate, exc); return n; } @@ -2144,16 +2148,7 @@ _PyGC_CollectNoFail(PyThreadState *tstate) during interpreter shutdown (and then never finish it). See http://bugs.python.org/issue8713#msg195178 for an example. */ - GCState *gcstate = &tstate->interp->gc; - if (gcstate->collecting) { - return 0; - } - - Py_ssize_t n; - gcstate->collecting = 1; - n = gc_collect_main(tstate, NUM_GENERATIONS - 1, NULL, NULL, 1); - gcstate->collecting = 0; - return n; + return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); } void @@ -2271,10 +2266,6 @@ PyObject_IS_GC(PyObject *obj) void _Py_ScheduleGC(PyInterpreterState *interp) { - GCState *gcstate = &interp->gc; - if (gcstate->collecting == 1) { - return; - } _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); } @@ -2292,7 +2283,7 @@ _PyObject_GC_Link(PyObject *op) if (gcstate->generations[0].count > gcstate->generations[0].threshold && gcstate->enabled && gcstate->generations[0].threshold && - !gcstate->collecting && + !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { _Py_ScheduleGC(tstate->interp); @@ -2302,10 +2293,7 @@ _PyObject_GC_Link(PyObject *op) void _Py_RunGC(PyThreadState *tstate) { - GCState *gcstate = &tstate->interp->gc; - gcstate->collecting = 1; - gc_collect_generations(tstate); - gcstate->collecting = 0; + gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); } static PyObject * diff --git a/Modules/getpath.c b/Modules/getpath.c index 6c1078b8914522..a3c8fc269d1c3c 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -22,7 +22,7 @@ #endif /* Reference the precompiled getpath.py */ -#include "../Python/frozen_modules/getpath.h" +#include "Python/frozen_modules/getpath.h" #if (!defined(PREFIX) || !defined(EXEC_PREFIX) \ || !defined(VERSION) || !defined(VPATH) \ @@ -502,6 +502,54 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) PyMem_Free((void *)path); PyMem_Free((void *)narrow); return r; +#elif defined(MS_WINDOWS) + HANDLE hFile; + wchar_t resolved[MAXPATHLEN+1]; + int len = 0, err; + Py_ssize_t pathlen; + PyObject *result; + + wchar_t *path = PyUnicode_AsWideCharString(pathobj, &pathlen); + if (!path) { + return NULL; + } + if (wcslen(path) != pathlen) { + PyErr_SetString(PyExc_ValueError, "path contains embedded nulls"); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + hFile = CreateFileW(path, 0, 0, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hFile != INVALID_HANDLE_VALUE) { + len = GetFinalPathNameByHandleW(hFile, resolved, MAXPATHLEN, VOLUME_NAME_DOS); + err = len ? 0 : GetLastError(); + CloseHandle(hFile); + } else { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + + if (err) { + PyErr_SetFromWindowsErr(err); + result = NULL; + } else if (len <= MAXPATHLEN) { + const wchar_t *p = resolved; + if (0 == wcsncmp(p, L"\\\\?\\", 4)) { + if (GetFileAttributesW(&p[4]) != INVALID_FILE_ATTRIBUTES) { + p += 4; + len -= 4; + } + } + if (CompareStringOrdinal(path, (int)pathlen, p, len, TRUE) == CSTR_EQUAL) { + result = Py_NewRef(pathobj); + } else { + result = PyUnicode_FromWideChar(p, len); + } + } else { + result = Py_NewRef(pathobj); + } + PyMem_Free(path); + return result; #endif return Py_NewRef(pathobj); diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 4ed34aa5bde827..164741495c7baf 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -105,20 +105,11 @@ class itertools.pairwise "pairwiseobject *" "clinic_state()->pairwise_type" /* batched object ************************************************************/ -/* Note: The built-in zip() function includes a "strict" argument - that was needed because that function would silently truncate data, - and there was no easy way for a user to detect the data loss. - The same reasoning does not apply to batched() which never drops data. - Instead, batched() produces a shorter tuple which can be handled - as the user sees fit. If requested, it would be reasonable to add - "fillvalue" support which had demonstrated value in zip_longest(). - For now, the API is kept simple and clean. - */ - typedef struct { PyObject_HEAD PyObject *it; Py_ssize_t batch_size; + bool strict; } batchedobject; /*[clinic input] @@ -126,6 +117,9 @@ typedef struct { itertools.batched.__new__ as batched_new iterable: object n: Py_ssize_t + * + strict: bool = False + Batch data into tuples of length n. The last batch may be shorter than n. Loops over the input iterable and accumulates data into tuples @@ -140,11 +134,15 @@ or when the input iterable is exhausted. ('D', 'E', 'F') ('G',) +If "strict" is True, raises a ValueError if the final batch is shorter +than n. + [clinic start generated code]*/ static PyObject * -batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n) -/*[clinic end generated code: output=7ebc954d655371b6 input=ffd70726927c5129]*/ +batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n, + int strict) +/*[clinic end generated code: output=c6de11b061529d3e input=7814b47e222f5467]*/ { PyObject *it; batchedobject *bo; @@ -170,6 +168,7 @@ batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n) } bo->batch_size = n; bo->it = it; + bo->strict = (bool) strict; return (PyObject *)bo; } @@ -233,6 +232,12 @@ batched_next(batchedobject *bo) Py_DECREF(result); return NULL; } + if (bo->strict) { + Py_CLEAR(bo->it); + Py_DECREF(result); + PyErr_SetString(PyExc_ValueError, "batched(): incomplete batch"); + return NULL; + } _PyTuple_Resize(&result, i); return result; } @@ -330,21 +335,30 @@ pairwise_next(pairwiseobject *po) return NULL; } if (old == NULL) { - po->old = old = (*Py_TYPE(it)->tp_iternext)(it); + old = (*Py_TYPE(it)->tp_iternext)(it); + Py_XSETREF(po->old, old); if (old == NULL) { Py_CLEAR(po->it); return NULL; } + it = po->it; + if (it == NULL) { + Py_CLEAR(po->old); + return NULL; + } } + Py_INCREF(old); new = (*Py_TYPE(it)->tp_iternext)(it); if (new == NULL) { Py_CLEAR(po->it); Py_CLEAR(po->old); + Py_DECREF(old); return NULL; } /* Future optimization: Reuse the result tuple as we do in enumerate() */ result = PyTuple_Pack(2, old, new); - Py_SETREF(po->old, new); + Py_XSETREF(po->old, new); + Py_DECREF(old); return result; } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d99b5335b6989a..39b1f3cb7b2b9b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -236,15 +236,16 @@ corresponding Unix manual entries for more information on calls."); # include #endif +#ifdef HAVE_SYS_TYPES_H +/* Should be included before on HP-UX v3 */ +# include +#endif /* HAVE_SYS_TYPES_H */ + #ifdef HAVE_SYS_SYSMACROS_H /* GNU C Library: major(), minor(), makedev() */ # include #endif -#ifdef HAVE_SYS_TYPES_H -# include -#endif /* HAVE_SYS_TYPES_H */ - #ifdef HAVE_SYS_STAT_H # include #endif /* HAVE_SYS_STAT_H */ @@ -1023,13 +1024,17 @@ typedef struct { PyObject *TerminalSizeType; PyObject *TimesResultType; PyObject *UnameResultType; -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) PyObject *WaitidResultType; #endif #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) PyObject *struct_rusage; #endif PyObject *st_mode; +#ifndef MS_WINDOWS + // times() clock frequency in hertz; used by os.times() + long ticks_per_second; +#endif } _posixstate; @@ -1553,6 +1558,7 @@ _Py_Sigset_Converter(PyObject *obj, void *addr) ** man environ(7). */ #include +#define USE_DARWIN_NS_GET_ENVIRON 1 #elif !defined(_MSC_VER) && (!defined(__WATCOMC__) || defined(__QNX__) || defined(__VXWORKS__)) extern char **environ; #endif /* !_MSC_VER */ @@ -1575,7 +1581,7 @@ convertenviron(void) through main() instead of wmain(). */ (void)_wgetenv(L""); e = _wenviron; -#elif defined(WITH_NEXT_FRAMEWORK) || (defined(__APPLE__) && defined(Py_ENABLE_SHARED)) +#elif defined(USE_DARWIN_NS_GET_ENVIRON) /* environ is not accessible as an extern in a shared object on OSX; use _NSGetEnviron to resolve it. The value changes if you add environment variables between calls to Py_Initialize, so don't cache the value. */ @@ -2286,7 +2292,7 @@ static PyStructSequence_Desc statvfs_result_desc = { 10 }; -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) PyDoc_STRVAR(waitid_result__doc__, "waitid_result: Result from waitid.\n\n\ This object may be accessed either as a tuple of\n\ @@ -2361,7 +2367,7 @@ _posix_clear(PyObject *module) Py_CLEAR(state->TerminalSizeType); Py_CLEAR(state->TimesResultType); Py_CLEAR(state->UnameResultType); -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) Py_CLEAR(state->WaitidResultType); #endif #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -2386,7 +2392,7 @@ _posix_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->TerminalSizeType); Py_VISIT(state->TimesResultType); Py_VISIT(state->UnameResultType); -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) Py_VISIT(state->WaitidResultType); #endif #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -2850,6 +2856,8 @@ FTRUNCATE #ifdef MS_WINDOWS #undef PATH_HAVE_FTRUNCATE #define PATH_HAVE_FTRUNCATE 1 + #undef PATH_HAVE_FCHMOD + #define PATH_HAVE_FCHMOD 1 #endif /*[python input] @@ -3305,6 +3313,60 @@ os_fchdir_impl(PyObject *module, int fd) } #endif /* HAVE_FCHDIR */ +#ifdef MS_WINDOWS +# define CHMOD_DEFAULT_FOLLOW_SYMLINKS 0 +#else +# define CHMOD_DEFAULT_FOLLOW_SYMLINKS 1 +#endif + +#ifdef MS_WINDOWS +static int +win32_lchmod(LPCWSTR path, int mode) +{ + DWORD attr = GetFileAttributesW(path); + if (attr == INVALID_FILE_ATTRIBUTES) { + return 0; + } + if (mode & _S_IWRITE) { + attr &= ~FILE_ATTRIBUTE_READONLY; + } + else { + attr |= FILE_ATTRIBUTE_READONLY; + } + return SetFileAttributesW(path, attr); +} + +static int +win32_hchmod(HANDLE hfile, int mode) +{ + FILE_BASIC_INFO info; + if (!GetFileInformationByHandleEx(hfile, FileBasicInfo, + &info, sizeof(info))) + { + return 0; + } + if (mode & _S_IWRITE) { + info.FileAttributes &= ~FILE_ATTRIBUTE_READONLY; + } + else { + info.FileAttributes |= FILE_ATTRIBUTE_READONLY; + } + return SetFileInformationByHandle(hfile, FileBasicInfo, + &info, sizeof(info)); +} + +static int +win32_fchmod(int fd, int mode) +{ + HANDLE hfile = _Py_get_osfhandle_noraise(fd); + if (hfile == INVALID_HANDLE_VALUE) { + SetLastError(ERROR_INVALID_HANDLE); + return 0; + } + return win32_hchmod(hfile, mode); +} + +#endif /* MS_WINDOWS */ /*[clinic input] os.chmod @@ -3327,7 +3389,8 @@ os.chmod and path should be relative; path will then be relative to that directory. - follow_symlinks: bool = True + follow_symlinks: bool(c_default="CHMOD_DEFAULT_FOLLOW_SYMLINKS", \ + py_default="(os.name != 'nt')") = CHMOD_DEFAULT_FOLLOW_SYMLINKS If False, and the last element of the path is a symbolic link, chmod will modify the symbolic link itself instead of the file the link points to. @@ -3344,20 +3407,16 @@ dir_fd and follow_symlinks may not be implemented on your platform. static PyObject * os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=5cf6a94915cc7bff input=674a14bc998de09d]*/ +/*[clinic end generated code: output=5cf6a94915cc7bff input=fcf115d174b9f3d8]*/ { int result; -#ifdef MS_WINDOWS - DWORD attr; -#endif - #ifdef HAVE_FCHMODAT int fchmodat_nofollow_unsupported = 0; int fchmodat_unsupported = 0; #endif -#if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD)) +#if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD) || defined(MS_WINDOWS)) if (follow_symlinks_specified("chmod", follow_symlinks)) return NULL; #endif @@ -3368,19 +3427,25 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, } #ifdef MS_WINDOWS + result = 0; Py_BEGIN_ALLOW_THREADS - attr = GetFileAttributesW(path->wide); - if (attr == INVALID_FILE_ATTRIBUTES) - result = 0; + if (path->fd != -1) { + result = win32_fchmod(path->fd, mode); + } + else if (follow_symlinks) { + HANDLE hfile = CreateFileW(path->wide, + FILE_READ_ATTRIBUTES|FILE_WRITE_ATTRIBUTES, + 0, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + result = win32_hchmod(hfile, mode); + (void)CloseHandle(hfile); + } + } else { - if (mode & _S_IWRITE) - attr &= ~FILE_ATTRIBUTE_READONLY; - else - attr |= FILE_ATTRIBUTE_READONLY; - result = SetFileAttributesW(path->wide, attr); + result = win32_lchmod(path->wide, mode); } Py_END_ALLOW_THREADS - if (!result) { return path_error(path); } @@ -3469,7 +3534,7 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, } -#ifdef HAVE_FCHMOD +#if defined(HAVE_FCHMOD) || defined(MS_WINDOWS) /*[clinic input] os.fchmod @@ -3491,12 +3556,21 @@ os_fchmod_impl(PyObject *module, int fd, int mode) /*[clinic end generated code: output=afd9bc05b4e426b3 input=b5594618bbbc22df]*/ { int res; - int async_err = 0; if (PySys_Audit("os.chmod", "iii", fd, mode, -1) < 0) { return NULL; } +#ifdef MS_WINDOWS + res = 0; + Py_BEGIN_ALLOW_THREADS + res = win32_fchmod(fd, mode); + Py_END_ALLOW_THREADS + if (!res) { + return PyErr_SetFromWindowsErr(0); + } +#else /* MS_WINDOWS */ + int async_err = 0; do { Py_BEGIN_ALLOW_THREADS res = fchmod(fd, mode); @@ -3504,13 +3578,14 @@ os_fchmod_impl(PyObject *module, int fd, int mode) } while (res != 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (res != 0) return (!async_err) ? posix_error() : NULL; +#endif /* MS_WINDOWS */ Py_RETURN_NONE; } -#endif /* HAVE_FCHMOD */ +#endif /* HAVE_FCHMOD || MS_WINDOWS */ -#ifdef HAVE_LCHMOD +#if defined(HAVE_LCHMOD) || defined(MS_WINDOWS) /*[clinic input] os.lchmod @@ -3531,6 +3606,15 @@ os_lchmod_impl(PyObject *module, path_t *path, int mode) if (PySys_Audit("os.chmod", "Oii", path->object, mode, -1) < 0) { return NULL; } +#ifdef MS_WINDOWS + Py_BEGIN_ALLOW_THREADS + res = win32_lchmod(path->wide, mode); + Py_END_ALLOW_THREADS + if (!res) { + path_error(path); + return NULL; + } +#else /* MS_WINDOWS */ Py_BEGIN_ALLOW_THREADS res = lchmod(path->narrow, mode); Py_END_ALLOW_THREADS @@ -3538,9 +3622,10 @@ os_lchmod_impl(PyObject *module, path_t *path, int mode) path_error(path); return NULL; } +#endif /* MS_WINDOWS */ Py_RETURN_NONE; } -#endif /* HAVE_LCHMOD */ +#endif /* HAVE_LCHMOD || MS_WINDOWS */ #ifdef HAVE_CHFLAGS @@ -6783,6 +6868,9 @@ enum posix_spawn_file_actions_identifier { POSIX_SPAWN_OPEN, POSIX_SPAWN_CLOSE, POSIX_SPAWN_DUP2 +#ifdef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP + ,POSIX_SPAWN_CLOSEFROM +#endif }; #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) @@ -7023,6 +7111,24 @@ parse_file_actions(PyObject *file_actions, } break; } +#ifdef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP + case POSIX_SPAWN_CLOSEFROM: { + int fd; + if (!PyArg_ParseTuple(file_action, "Oi" + ";A closefrom file_action tuple must have 2 elements", + &tag_obj, &fd)) + { + goto fail; + } + errno = posix_spawn_file_actions_addclosefrom_np(file_actionsp, + fd); + if (errno) { + posix_error(); + goto fail; + } + break; + } +#endif default: { PyErr_SetString(PyExc_TypeError, "Unknown file_actions identifier"); @@ -7078,9 +7184,9 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a return NULL; } - if (!PyMapping_Check(env)) { + if (!PyMapping_Check(env) && env != Py_None) { PyErr_Format(PyExc_TypeError, - "%s: environment must be a mapping object", func_name); + "%s: environment must be a mapping object or None", func_name); goto exit; } @@ -7094,9 +7200,21 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a goto exit; } - envlist = parse_envlist(env, &envc); - if (envlist == NULL) { - goto exit; +#ifdef USE_DARWIN_NS_GET_ENVIRON + // There is no environ global in this situation. + char **environ = NULL; +#endif + + if (env == Py_None) { +#ifdef USE_DARWIN_NS_GET_ENVIRON + environ = *_NSGetEnviron(); +#endif + envlist = environ; + } else { + envlist = parse_envlist(env, &envc); + if (envlist == NULL) { + goto exit; + } } if (file_actions != NULL && file_actions != Py_None) { @@ -7159,7 +7277,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a if (attrp) { (void)posix_spawnattr_destroy(attrp); } - if (envlist) { + if (envlist && envlist != environ) { free_string_array(envlist, envc); } if (argvlist) { @@ -9400,7 +9518,7 @@ os_wait4_impl(PyObject *module, pid_t pid, int options) #endif /* HAVE_WAIT4 */ -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) /*[clinic input] os.waitid @@ -9457,7 +9575,7 @@ os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options) return result; } -#endif /* defined(HAVE_WAITID) && !defined(__APPLE__) */ +#endif /* defined(HAVE_WAITID) */ #if defined(HAVE_WAITPID) @@ -9986,8 +10104,6 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, #endif /* HAVE_SYMLINK */ - - static PyStructSequence_Field times_result_fields[] = { {"user", "user time"}, {"system", "system time"}, @@ -10013,12 +10129,6 @@ static PyStructSequence_Desc times_result_desc = { 5 }; -#ifdef MS_WINDOWS -#define HAVE_TIMES /* mandatory, for the method table */ -#endif - -#ifdef HAVE_TIMES - static PyObject * build_times_result(PyObject *module, double user, double system, double children_user, double children_system, @@ -10064,8 +10174,8 @@ All fields are floating point numbers. static PyObject * os_times_impl(PyObject *module) /*[clinic end generated code: output=35f640503557d32a input=2bf9df3d6ab2e48b]*/ -#ifdef MS_WINDOWS { +#ifdef MS_WINDOWS FILETIME create, exit, kernel, user; HANDLE hProc; hProc = GetCurrentProcess(); @@ -10083,28 +10193,26 @@ os_times_impl(PyObject *module) (double)0, (double)0, (double)0); -} #else /* MS_WINDOWS */ -{ - struct tms t; - clock_t c; + _posixstate *state = get_posix_state(module); + long ticks_per_second = state->ticks_per_second; + + struct tms process; + clock_t elapsed; errno = 0; - c = times(&t); - if (c == (clock_t) -1) { + elapsed = times(&process); + if (elapsed == (clock_t) -1) { return posix_error(); } - assert(_PyRuntime.time.ticks_per_second_initialized); -#define ticks_per_second _PyRuntime.time.ticks_per_second + return build_times_result(module, - (double)t.tms_utime / ticks_per_second, - (double)t.tms_stime / ticks_per_second, - (double)t.tms_cutime / ticks_per_second, - (double)t.tms_cstime / ticks_per_second, - (double)c / ticks_per_second); -#undef ticks_per_second -} + (double)process.tms_utime / ticks_per_second, + (double)process.tms_stime / ticks_per_second, + (double)process.tms_cutime / ticks_per_second, + (double)process.tms_cstime / ticks_per_second, + (double)elapsed / ticks_per_second); #endif /* MS_WINDOWS */ -#endif /* HAVE_TIMES */ +} #if defined(HAVE_TIMERFD_CREATE) @@ -16079,6 +16187,26 @@ os_waitstatus_to_exitcode_impl(PyObject *module, PyObject *status_obj) } #endif +#if defined(MS_WINDOWS) +/*[clinic input] +os._supports_virtual_terminal + +Checks if virtual terminal is supported in windows +[clinic start generated code]*/ + +static PyObject * +os__supports_virtual_terminal_impl(PyObject *module) +/*[clinic end generated code: output=bd0556a6d9d99fe6 input=0752c98e5d321542]*/ +{ + DWORD mode = 0; + HANDLE handle = GetStdHandle(STD_ERROR_HANDLE); + if (!GetConsoleMode(handle, &mode)) { + Py_RETURN_FALSE; + } + return PyBool_FromLong(mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING); +} +#endif + static PyMethodDef posix_methods[] = { @@ -16283,6 +16411,8 @@ static PyMethodDef posix_methods[] = { OS__PATH_ISFILE_METHODDEF OS__PATH_ISLINK_METHODDEF OS__PATH_EXISTS_METHODDEF + + OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF {NULL, NULL} /* Sentinel */ }; @@ -16707,6 +16837,9 @@ all_ins(PyObject *m) if (PyModule_AddIntConstant(m, "POSIX_SPAWN_OPEN", POSIX_SPAWN_OPEN)) return -1; if (PyModule_AddIntConstant(m, "POSIX_SPAWN_CLOSE", POSIX_SPAWN_CLOSE)) return -1; if (PyModule_AddIntConstant(m, "POSIX_SPAWN_DUP2", POSIX_SPAWN_DUP2)) return -1; +#ifdef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP + if (PyModule_AddIntMacro(m, POSIX_SPAWN_CLOSEFROM)) return -1; +#endif #endif #if defined(HAVE_SPAWNV) || defined (HAVE_RTPSPAWN) @@ -17176,7 +17309,7 @@ posixmodule_exec(PyObject *m) return -1; } -#if defined(HAVE_WAITID) && !defined(__APPLE__) +#if defined(HAVE_WAITID) waitid_result_desc.name = MODNAME ".waitid_result"; state->WaitidResultType = (PyObject *)PyStructSequence_NewType(&waitid_result_desc); if (PyModule_AddObjectRef(m, "waitid_result", state->WaitidResultType) < 0) { @@ -17279,6 +17412,15 @@ posixmodule_exec(PyObject *m) Py_DECREF(unicode); } +#ifndef MS_WINDOWS + if (_Py_GetTicksPerSecond(&state->ticks_per_second) < 0) { + PyErr_SetString(PyExc_RuntimeError, + "cannot read ticks_per_second"); + return -1; + } + assert(state->ticks_per_second >= 1); +#endif + return PyModule_Add(m, "_have_functions", list); } diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 21579a80dd7f70..ec44892d101e44 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -240,19 +240,12 @@ string_intern(xmlparseobject *self, const char* str) return result; if (!self->intern) return result; - value = PyDict_GetItemWithError(self->intern, result); - if (!value) { - if (!PyErr_Occurred() && - PyDict_SetItem(self->intern, result, result) == 0) - { - return result; - } - else { - Py_DECREF(result); - return NULL; - } + if (PyDict_GetItemRef(self->intern, result, &value) == 0 && + PyDict_SetItem(self->intern, result, result) == 0) + { + return result; } - Py_INCREF(value); + assert((value != NULL) == !PyErr_Occurred()); Py_DECREF(result); return value; } @@ -2069,9 +2062,7 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/readline.c b/Modules/readline.c index fde552d124bc77..e29051c37f8827 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -147,8 +147,19 @@ readline_free(void *m) static PyModuleDef readlinemodule; -#define readlinestate_global ((readlinestate *)PyModule_GetState(PyState_FindModule(&readlinemodule))) - +static inline readlinestate* +get_hook_module_state(void) +{ + PyObject *mod = PyState_FindModule(&readlinemodule); + if (mod == NULL){ + PyErr_Clear(); + return NULL; + } + Py_INCREF(mod); + readlinestate *state = get_readline_state(mod); + Py_DECREF(mod); + return state; +} /* Convert to/from multibyte C strings */ @@ -438,14 +449,15 @@ readline_set_completion_display_matches_hook_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=516e5cb8db75a328 input=4f0bfd5ab0179a26]*/ { + readlinestate *state = get_readline_state(module); PyObject *result = set_hook("completion_display_matches_hook", - &readlinestate_global->completion_display_matches_hook, + &state->completion_display_matches_hook, function); #ifdef HAVE_RL_COMPLETION_DISPLAY_MATCHES_HOOK /* We cannot set this hook globally, since it replaces the default completion display. */ rl_completion_display_matches_hook = - readlinestate_global->completion_display_matches_hook ? + state->completion_display_matches_hook ? #if defined(HAVE_RL_COMPDISP_FUNC_T) (rl_compdisp_func_t *)on_completion_display_matches_hook : 0; #else @@ -472,7 +484,8 @@ static PyObject * readline_set_startup_hook_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=02cd0e0c4fa082ad input=7783b4334b26d16d]*/ { - return set_hook("startup_hook", &readlinestate_global->startup_hook, + readlinestate *state = get_readline_state(module); + return set_hook("startup_hook", &state->startup_hook, function); } @@ -497,7 +510,8 @@ static PyObject * readline_set_pre_input_hook_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=fe1a96505096f464 input=4f3eaeaf7ce1fdbe]*/ { - return set_hook("pre_input_hook", &readlinestate_global->pre_input_hook, + readlinestate *state = get_readline_state(module); + return set_hook("pre_input_hook", &state->pre_input_hook, function); } #endif @@ -530,7 +544,8 @@ static PyObject * readline_get_begidx_impl(PyObject *module) /*[clinic end generated code: output=362616ee8ed1b2b1 input=e083b81c8eb4bac3]*/ { - return Py_NewRef(readlinestate_global->begidx); + readlinestate *state = get_readline_state(module); + return Py_NewRef(state->begidx); } /* Get the ending index for the scope of the tab-completion */ @@ -545,7 +560,8 @@ static PyObject * readline_get_endidx_impl(PyObject *module) /*[clinic end generated code: output=7f763350b12d7517 input=d4c7e34a625fd770]*/ { - return Py_NewRef(readlinestate_global->endidx); + readlinestate *state = get_readline_state(module); + return Py_NewRef(state->endidx); } /* Set the tab-completion word-delimiters that readline uses */ @@ -576,6 +592,13 @@ readline_set_completer_delims(PyObject *module, PyObject *string) if (break_chars) { free(completer_word_break_characters); completer_word_break_characters = break_chars; +#ifdef WITH_EDITLINE + rl_basic_word_break_characters = break_chars; +#else + if (using_libedit_emulation) { + rl_basic_word_break_characters = break_chars; + } +#endif rl_completer_word_break_characters = break_chars; Py_RETURN_NONE; } @@ -772,7 +795,8 @@ static PyObject * readline_set_completer_impl(PyObject *module, PyObject *function) /*[clinic end generated code: output=171a2a60f81d3204 input=51e81e13118eb877]*/ { - return set_hook("completer", &readlinestate_global->completer, function); + readlinestate *state = get_readline_state(module); + return set_hook("completer", &state->completer, function); } /*[clinic input] @@ -785,10 +809,11 @@ static PyObject * readline_get_completer_impl(PyObject *module) /*[clinic end generated code: output=6e6bbd8226d14475 input=6457522e56d70d13]*/ { - if (readlinestate_global->completer == NULL) { + readlinestate *state = get_readline_state(module); + if (state->completer == NULL) { Py_RETURN_NONE; } - return Py_NewRef(readlinestate_global->completer); + return Py_NewRef(state->completer); } /* Private function to get current length of history. XXX It may be @@ -1018,15 +1043,18 @@ on_hook(PyObject *func) static int #if defined(_RL_FUNCTION_TYPEDEF) on_startup_hook(void) -#elif defined(WITH_APPLE_EDITLINE) -on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #else -on_startup_hook(void) +on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #endif { int r; PyGILState_STATE gilstate = PyGILState_Ensure(); - r = on_hook(readlinestate_global->startup_hook); + readlinestate *state = get_hook_module_state(); + if (state == NULL) { + PyGILState_Release(gilstate); + return -1; + } + r = on_hook(state->startup_hook); PyGILState_Release(gilstate); return r; } @@ -1035,15 +1063,18 @@ on_startup_hook(void) static int #if defined(_RL_FUNCTION_TYPEDEF) on_pre_input_hook(void) -#elif defined(WITH_APPLE_EDITLINE) -on_pre_input_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #else -on_pre_input_hook(void) +on_pre_input_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #endif { int r; PyGILState_STATE gilstate = PyGILState_Ensure(); - r = on_hook(readlinestate_global->pre_input_hook); + readlinestate *state = get_hook_module_state(); + if (state == NULL) { + PyGILState_Release(gilstate); + return -1; + } + r = on_hook(state->pre_input_hook); PyGILState_Release(gilstate); return r; } @@ -1060,6 +1091,11 @@ on_completion_display_matches_hook(char **matches, int i; PyObject *sub, *m=NULL, *s=NULL, *r=NULL; PyGILState_STATE gilstate = PyGILState_Ensure(); + readlinestate *state = get_hook_module_state(); + if (state == NULL) { + PyGILState_Release(gilstate); + return; + } m = PyList_New(num_matches); if (m == NULL) goto error; @@ -1070,7 +1106,7 @@ on_completion_display_matches_hook(char **matches, PyList_SET_ITEM(m, i, s); } sub = decode(matches[0]); - r = PyObject_CallFunction(readlinestate_global->completion_display_matches_hook, + r = PyObject_CallFunction(state->completion_display_matches_hook, "NNi", sub, m, max_length); m=NULL; @@ -1118,12 +1154,17 @@ static char * on_completion(const char *text, int state) { char *result = NULL; - if (readlinestate_global->completer != NULL) { + PyGILState_STATE gilstate = PyGILState_Ensure(); + readlinestate *module_state = get_hook_module_state(); + if (module_state == NULL) { + PyGILState_Release(gilstate); + return NULL; + } + if (module_state->completer != NULL) { PyObject *r = NULL, *t; - PyGILState_STATE gilstate = PyGILState_Ensure(); rl_attempted_completion_over = 1; t = decode(text); - r = PyObject_CallFunction(readlinestate_global->completer, "Ni", t, state); + r = PyObject_CallFunction(module_state->completer, "Ni", t, state); if (r == NULL) goto error; if (r == Py_None) { @@ -1145,6 +1186,7 @@ on_completion(const char *text, int state) PyGILState_Release(gilstate); return result; } + PyGILState_Release(gilstate); return result; } @@ -1160,6 +1202,7 @@ flex_complete(const char *text, int start, int end) size_t start_size, end_size; wchar_t *s; PyGILState_STATE gilstate = PyGILState_Ensure(); + readlinestate *state = get_hook_module_state(); #ifdef HAVE_RL_COMPLETION_APPEND_CHARACTER rl_completion_append_character ='\0'; #endif @@ -1187,10 +1230,12 @@ flex_complete(const char *text, int start, int end) end = start + (int)end_size; done: - Py_XDECREF(readlinestate_global->begidx); - Py_XDECREF(readlinestate_global->endidx); - readlinestate_global->begidx = PyLong_FromLong((long) start); - readlinestate_global->endidx = PyLong_FromLong((long) end); + if (state) { + Py_XDECREF(state->begidx); + Py_XDECREF(state->endidx); + state->begidx = PyLong_FromLong((long) start); + state->endidx = PyLong_FromLong((long) end); + } result = completion_matches((char *)text, *on_completion); PyGILState_Release(gilstate); return result; @@ -1267,6 +1312,15 @@ setup_readline(readlinestate *mod_state) completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ +#ifdef WITH_EDITLINE + // libedit uses rl_basic_word_break_characters instead of + // rl_completer_word_break_characters as complete delimiter + rl_basic_word_break_characters = completer_word_break_characters; +#else + if (using_libedit_emulation) { + rl_basic_word_break_characters = completer_word_break_characters; + } +#endif rl_completer_word_break_characters = completer_word_break_characters; mod_state->begidx = PyLong_FromLong(0L); @@ -1480,6 +1534,7 @@ static struct PyModuleDef readlinemodule = { PyMODINIT_FUNC PyInit_readline(void) { + const char *backend = "readline"; PyObject *m; readlinestate *mod_state; @@ -1487,8 +1542,10 @@ PyInit_readline(void) using_libedit_emulation = 1; } - if (using_libedit_emulation) + if (using_libedit_emulation) { readlinemodule.m_doc = doc_module_le; + backend = "editline"; + } m = PyModule_Create(&readlinemodule); @@ -1510,13 +1567,22 @@ PyInit_readline(void) goto error; } + if (PyModule_AddStringConstant(m, "backend", backend) < 0) { + goto error; + } + mod_state = (readlinestate *) PyModule_GetState(m); + if (mod_state == NULL){ + goto error; + } PyOS_ReadlineFunctionPointer = call_readline; if (setup_readline(mod_state) < 0) { PyErr_NoMemory(); goto error; } - + if (PyErr_Occurred()){ + goto error; + } return m; error: diff --git a/Modules/resource.c b/Modules/resource.c index a4b8f648c329e3..19020b8cc1b6db 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,6 +1,4 @@ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.13 for PySys_Audit() diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 9ac2001c0132d3..0a0e0e78656f76 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7071,17 +7071,23 @@ _socket_socket_if_nametoindex_impl(PySocketSockObject *self, PyObject *oname) static PyObject * socket_if_indextoname(PyObject *self, PyObject *arg) { + unsigned long index_long = PyLong_AsUnsignedLong(arg); + if (index_long == (unsigned long) -1 && PyErr_Occurred()) { + return NULL; + } + #ifdef MS_WINDOWS - NET_IFINDEX index; + NET_IFINDEX index = (NET_IFINDEX)index_long; #else - unsigned long index; + unsigned int index = (unsigned int)index_long; #endif - char name[IF_NAMESIZE + 1]; - index = PyLong_AsUnsignedLong(arg); - if (index == (unsigned long) -1) + if ((unsigned long)index != index_long) { + PyErr_SetString(PyExc_OverflowError, "index is too large"); return NULL; + } + char name[IF_NAMESIZE + 1]; if (if_indextoname(index, name) == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 6a44850e291448..62c7816f891ee2 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -406,6 +406,30 @@ syslog_exec(PyObject *module) ADD_INT_MACRO(module, LOG_AUTHPRIV); #endif +#ifdef LOG_FTP + ADD_INT_MACRO(module, LOG_FTP); +#endif + +#ifdef LOG_NETINFO + ADD_INT_MACRO(module, LOG_NETINFO); +#endif + +#ifdef LOG_REMOTEAUTH + ADD_INT_MACRO(module, LOG_REMOTEAUTH); +#endif + +#ifdef LOG_INSTALL + ADD_INT_MACRO(module, LOG_INSTALL); +#endif + +#ifdef LOG_RAS + ADD_INT_MACRO(module, LOG_RAS); +#endif + +#ifdef LOG_LAUNCHD + ADD_INT_MACRO(module, LOG_LAUNCHD); +#endif + return 0; } diff --git a/Modules/termios.c b/Modules/termios.c index 9fc2673ce0e788..c4f0fd9d50044a 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -702,6 +702,9 @@ static struct constant { #ifdef IMAXBEL {"IMAXBEL", IMAXBEL}, #endif +#ifdef IUTF8 + {"IUTF8", IUTF8}, +#endif /* struct termios.c_oflag constants */ {"OPOST", OPOST}, @@ -726,6 +729,12 @@ static struct constant { #ifdef OFDEL {"OFDEL", OFDEL}, #endif +#ifdef OXTABS + {"OXTABS", OXTABS}, +#endif +#ifdef ONOEOT + {"ONOEOT", ONOEOT}, +#endif #ifdef NLDLY {"NLDLY", NLDLY}, #endif @@ -752,6 +761,12 @@ static struct constant { #ifdef NL1 {"NL1", NL1}, #endif +#ifdef NL2 + {"NL2", NL2}, +#endif +#ifdef NL3 + {"NL3", NL3}, +#endif #ifdef CR0 {"CR0", CR0}, #endif @@ -799,6 +814,9 @@ static struct constant { #endif /* struct termios.c_cflag constants */ +#ifdef CIGNORE + {"CIGNORE", CIGNORE}, +#endif {"CSIZE", CSIZE}, {"CSTOPB", CSTOPB}, {"CREAD", CREAD}, @@ -813,6 +831,25 @@ static struct constant { {"CRTSCTS", (long)CRTSCTS}, #endif +#ifdef CRTS_IFLOW + {"CRTS_IFLOW", CRTS_IFLOW}, +#endif +#ifdef CDTR_IFLOW + {"CDTR_IFLOW", CDTR_IFLOW}, +#endif +#ifdef CDSR_OFLOW + {"CDSR_OFLOW", CDSR_OFLOW}, +#endif +#ifdef CCTS_OFLOW + {"CCTS_OFLOW", CCTS_OFLOW}, +#endif +#ifdef CCAR_OFLOW + {"CCAR_OFLOW", CCAR_OFLOW}, +#endif +#ifdef MDMBUF + {"MDMBUF", MDMBUF}, +#endif + /* struct termios.c_cflag-related values (character size) */ {"CS5", CS5}, {"CS6", CS6}, @@ -820,6 +857,9 @@ static struct constant { {"CS8", CS8}, /* struct termios.c_lflag constants */ +#ifdef ALTWERASE + {"ALTWERASE", ALTWERASE}, +#endif {"ISIG", ISIG}, {"ICANON", ICANON}, #ifdef XCASE @@ -840,6 +880,9 @@ static struct constant { #endif #ifdef FLUSHO {"FLUSHO", FLUSHO}, +#endif +#ifdef NOKERNINFO + {"NOKERNINFO", NOKERNINFO}, #endif {"NOFLSH", NOFLSH}, {"TOSTOP", TOSTOP}, @@ -847,6 +890,9 @@ static struct constant { {"PENDIN", PENDIN}, #endif {"IEXTEN", IEXTEN}, +#ifdef EXTPROC + {"EXTPROC", EXTPROC}, +#endif /* indexes into the control chars array returned by tcgetattr() */ {"VINTR", VINTR}, @@ -855,6 +901,9 @@ static struct constant { {"VKILL", VKILL}, {"VEOF", VEOF}, {"VTIME", VTIME}, +#ifdef VSTATUS + {"VSTATUS", VSTATUS}, +#endif {"VMIN", VMIN}, #ifdef VSWTC /* The #defines above ensure that if either is defined, both are, @@ -865,6 +914,9 @@ static struct constant { {"VSTART", VSTART}, {"VSTOP", VSTOP}, {"VSUSP", VSUSP}, +#ifdef VDSUSP + {"VDSUSP", VDSUSP}, +#endif {"VEOL", VEOL}, #ifdef VREPRINT {"VREPRINT", VREPRINT}, @@ -883,6 +935,18 @@ static struct constant { #endif +#ifdef B7200 + {"B7200", B7200}, +#endif +#ifdef B14400 + {"B14400", B14400}, +#endif +#ifdef B28800 + {"B28800", B28800}, +#endif +#ifdef B76800 + {"B76800", B76800}, +#endif #ifdef B460800 {"B460800", B460800}, #endif diff --git a/Modules/timemodule.c b/Modules/timemodule.c index bc3901e0d7a621..b3fe175d9b184a 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -69,62 +69,20 @@ module time /*[clinic end generated code: output=da39a3ee5e6b4b0d input=a668a08771581f36]*/ -#if defined(HAVE_TIMES) || defined(HAVE_CLOCK) -static int -check_ticks_per_second(long tps, const char *context) -{ - /* Effectively, check that _PyTime_MulDiv(t, SEC_TO_NS, ticks_per_second) - cannot overflow. */ - if (tps >= 0 && (_PyTime_t)tps > _PyTime_MAX / SEC_TO_NS) { - PyErr_Format(PyExc_OverflowError, "%s is too large", context); - return -1; - } - return 0; -} -#endif /* HAVE_TIMES || HAVE_CLOCK */ - -#ifdef HAVE_TIMES - -# define ticks_per_second _PyRuntime.time.ticks_per_second - -static void -ensure_ticks_per_second(void) -{ - if (_PyRuntime.time.ticks_per_second_initialized) { - return; - } - _PyRuntime.time.ticks_per_second_initialized = 1; -# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - ticks_per_second = sysconf(_SC_CLK_TCK); - if (ticks_per_second < 1) { - ticks_per_second = -1; - } -# elif defined(HZ) - ticks_per_second = HZ; -# else - ticks_per_second = 60; /* magic fallback value; may be bogus */ -# endif -} - -#endif /* HAVE_TIMES */ - - -PyStatus -_PyTime_Init(void) -{ -#ifdef HAVE_TIMES - ensure_ticks_per_second(); -#endif - return PyStatus_Ok(); -} - - /* Forward declarations */ static int pysleep(_PyTime_t timeout); typedef struct { PyTypeObject *struct_time_type; +#ifdef HAVE_TIMES + // times() clock frequency in hertz + _PyTimeFraction times_base; +#endif +#ifdef HAVE_CLOCK + // clock() frequency in hertz + _PyTimeFraction clock_base; +#endif } time_module_state; static inline time_module_state* @@ -184,7 +142,7 @@ PyDoc_STRVAR(time_ns_doc, \n\ Return the current time in nanoseconds since the Epoch."); -#if defined(HAVE_CLOCK) +#ifdef HAVE_CLOCK #ifndef CLOCKS_PER_SEC # ifdef CLK_TCK @@ -195,15 +153,13 @@ Return the current time in nanoseconds since the Epoch."); #endif static int -_PyTime_GetClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +py_clock(time_module_state *state, _PyTime_t *tp, _Py_clock_info_t *info) { - if (check_ticks_per_second(CLOCKS_PER_SEC, "CLOCKS_PER_SEC") < 0) { - return -1; - } + _PyTimeFraction *base = &state->clock_base; if (info) { info->implementation = "clock()"; - info->resolution = 1.0 / (double)CLOCKS_PER_SEC; + info->resolution = _PyTimeFraction_Resolution(base); info->monotonic = 1; info->adjustable = 0; } @@ -215,7 +171,7 @@ _PyTime_GetClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) "or its value cannot be represented"); return -1; } - _PyTime_t ns = _PyTime_MulDiv(ticks, SEC_TO_NS, (_PyTime_t)CLOCKS_PER_SEC); + _PyTime_t ns = _PyTimeFraction_Mul(ticks, base); *tp = _PyTime_FromNanoseconds(ns); return 0; } @@ -1277,8 +1233,38 @@ PyDoc_STRVAR(perf_counter_ns_doc, \n\ Performance counter for benchmarking as nanoseconds."); + +#ifdef HAVE_TIMES static int -_PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +process_time_times(time_module_state *state, _PyTime_t *tp, + _Py_clock_info_t *info) +{ + _PyTimeFraction *base = &state->times_base; + + struct tms process; + if (times(&process) == (clock_t)-1) { + return 0; + } + + if (info) { + info->implementation = "times()"; + info->resolution = _PyTimeFraction_Resolution(base); + info->monotonic = 1; + info->adjustable = 0; + } + + _PyTime_t ns; + ns = _PyTimeFraction_Mul(process.tms_utime, base); + ns += _PyTimeFraction_Mul(process.tms_stime, base); + *tp = _PyTime_FromNanoseconds(ns); + return 1; +} +#endif + + +static int +py_process_time(time_module_state *state, _PyTime_t *tp, + _Py_clock_info_t *info) { #if defined(MS_WINDOWS) HANDLE process; @@ -1381,41 +1367,27 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) /* times() */ #ifdef HAVE_TIMES - struct tms t; - - if (times(&t) != (clock_t)-1) { - assert(_PyRuntime.time.ticks_per_second_initialized); - if (check_ticks_per_second(ticks_per_second, "_SC_CLK_TCK") < 0) { - return -1; - } - if (ticks_per_second != -1) { - if (info) { - info->implementation = "times()"; - info->monotonic = 1; - info->adjustable = 0; - info->resolution = 1.0 / (double)ticks_per_second; - } - - _PyTime_t ns; - ns = _PyTime_MulDiv(t.tms_utime, SEC_TO_NS, ticks_per_second); - ns += _PyTime_MulDiv(t.tms_stime, SEC_TO_NS, ticks_per_second); - *tp = _PyTime_FromNanoseconds(ns); - return 0; - } + int res = process_time_times(state, tp, info); + if (res < 0) { + return -1; } + if (res == 1) { + return 0; + } + // times() failed, ignore failure #endif - /* clock */ - /* Currently, Python 3 requires clock() to build: see issue #22624 */ - return _PyTime_GetClockWithInfo(tp, info); + /* clock(). Python 3 requires clock() to build (see gh-66814) */ + return py_clock(state, tp, info); #endif } static PyObject * -time_process_time(PyObject *self, PyObject *unused) +time_process_time(PyObject *module, PyObject *unused) { + time_module_state *state = get_time_state(module); _PyTime_t t; - if (_PyTime_GetProcessTimeWithInfo(&t, NULL) < 0) { + if (py_process_time(state, &t, NULL) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1427,10 +1399,11 @@ PyDoc_STRVAR(process_time_doc, Process time for profiling: sum of the kernel and user-space CPU time."); static PyObject * -time_process_time_ns(PyObject *self, PyObject *unused) +time_process_time_ns(PyObject *module, PyObject *unused) { + time_module_state *state = get_time_state(module); _PyTime_t t; - if (_PyTime_GetProcessTimeWithInfo(&t, NULL) < 0) { + if (py_process_time(state, &t, NULL) < 0) { return NULL; } return _PyTime_AsNanosecondsObject(t); @@ -1617,7 +1590,7 @@ sum of the kernel and user-space CPU time."); static PyObject * -time_get_clock_info(PyObject *self, PyObject *args) +time_get_clock_info(PyObject *module, PyObject *args) { char *name; _Py_clock_info_t info; @@ -1656,7 +1629,8 @@ time_get_clock_info(PyObject *self, PyObject *args) } } else if (strcmp(name, "process_time") == 0) { - if (_PyTime_GetProcessTimeWithInfo(&t, &info) < 0) { + time_module_state *state = get_time_state(module); + if (py_process_time(state, &t, &info) < 0) { return NULL; } } @@ -2116,6 +2090,28 @@ time_exec(PyObject *module) } #endif +#ifdef HAVE_TIMES + long ticks_per_second; + if (_Py_GetTicksPerSecond(&ticks_per_second) < 0) { + PyErr_SetString(PyExc_RuntimeError, + "cannot read ticks_per_second"); + return -1; + } + if (_PyTimeFraction_Set(&state->times_base, SEC_TO_NS, + ticks_per_second) < 0) { + PyErr_Format(PyExc_OverflowError, "ticks_per_second is too large"); + return -1; + } +#endif + +#ifdef HAVE_CLOCK + if (_PyTimeFraction_Set(&state->clock_base, SEC_TO_NS, + CLOCKS_PER_SEC) < 0) { + PyErr_Format(PyExc_OverflowError, "CLOCKS_PER_SEC is too large"); + return -1; + } +#endif + return 0; } diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 19f61216255cfa..0bb5e12d7c3dd9 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -62,9 +62,7 @@ pass */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 867820a6cb93fa..754a368f77e940 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -5,9 +5,7 @@ * See the xxlimited module for an extension module template. */ -#ifndef _MSC_VER #include "pyconfig.h" // Py_GIL_DISABLED -#endif #ifndef Py_GIL_DISABLED #define Py_LIMITED_API 0x03050000 diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 9b76afa0e56f76..fe9a6d8d4150ab 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1896,12 +1896,20 @@ zlib_crc32_impl(PyObject *module, Py_buffer *data, unsigned int value) Py_BEGIN_ALLOW_THREADS /* Avoid truncation of length for very large buffers. crc32() takes - length as an unsigned int, which may be narrower than Py_ssize_t. */ - while ((size_t)len > UINT_MAX) { - value = crc32(value, buf, UINT_MAX); - buf += (size_t) UINT_MAX; - len -= (size_t) UINT_MAX; + length as an unsigned int, which may be narrower than Py_ssize_t. + We further limit size due to bugs in Apple's macOS zlib. + See https://github.com/python/cpython/issues/105967. + */ +#define ZLIB_CRC_CHUNK_SIZE 0x40000000 +#if ZLIB_CRC_CHUNK_SIZE > INT_MAX +# error "unsupported less than 32-bit platform?" +#endif + while ((size_t)len > ZLIB_CRC_CHUNK_SIZE) { + value = crc32(value, buf, ZLIB_CRC_CHUNK_SIZE); + buf += (size_t) ZLIB_CRC_CHUNK_SIZE; + len -= (size_t) ZLIB_CRC_CHUNK_SIZE; } +#undef ZLIB_CRC_CHUNK_SIZE value = crc32(value, buf, (unsigned int)len); Py_END_ALLOW_THREADS } else { diff --git a/Objects/abstract.c b/Objects/abstract.c index 43842fbdd6aedd..1ec5c5b8c3dc2f 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1180,29 +1180,10 @@ PyNumber_Multiply(PyObject *v, PyObject *w) return result; } -PyObject * -PyNumber_MatrixMultiply(PyObject *v, PyObject *w) -{ - return binary_op(v, w, NB_SLOT(nb_matrix_multiply), "@"); -} - -PyObject * -PyNumber_FloorDivide(PyObject *v, PyObject *w) -{ - return binary_op(v, w, NB_SLOT(nb_floor_divide), "//"); -} - -PyObject * -PyNumber_TrueDivide(PyObject *v, PyObject *w) -{ - return binary_op(v, w, NB_SLOT(nb_true_divide), "/"); -} - -PyObject * -PyNumber_Remainder(PyObject *v, PyObject *w) -{ - return binary_op(v, w, NB_SLOT(nb_remainder), "%"); -} +BINARY_FUNC(PyNumber_MatrixMultiply, nb_matrix_multiply, "@") +BINARY_FUNC(PyNumber_FloorDivide, nb_floor_divide, "//") +BINARY_FUNC(PyNumber_TrueDivide, nb_true_divide, "/") +BINARY_FUNC(PyNumber_Remainder, nb_remainder, "%") PyObject * PyNumber_Power(PyObject *v, PyObject *w, PyObject *z) @@ -1379,73 +1360,27 @@ _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs) /* Unary operators and functions */ -PyObject * -PyNumber_Negative(PyObject *o) -{ - if (o == NULL) { - return null_error(); - } - - PyNumberMethods *m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_negative) { - PyObject *res = (*m->nb_negative)(o); - assert(_Py_CheckSlotResult(o, "__neg__", res != NULL)); - return res; - } - - return type_error("bad operand type for unary -: '%.200s'", o); -} - -PyObject * -PyNumber_Positive(PyObject *o) -{ - if (o == NULL) { - return null_error(); - } - - PyNumberMethods *m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_positive) { - PyObject *res = (*m->nb_positive)(o); - assert(_Py_CheckSlotResult(o, "__pos__", res != NULL)); - return res; - } - - return type_error("bad operand type for unary +: '%.200s'", o); -} - -PyObject * -PyNumber_Invert(PyObject *o) -{ - if (o == NULL) { - return null_error(); - } - - PyNumberMethods *m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_invert) { - PyObject *res = (*m->nb_invert)(o); - assert(_Py_CheckSlotResult(o, "__invert__", res != NULL)); - return res; - } - - return type_error("bad operand type for unary ~: '%.200s'", o); -} - -PyObject * -PyNumber_Absolute(PyObject *o) -{ - if (o == NULL) { - return null_error(); - } - - PyNumberMethods *m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_absolute) { - PyObject *res = m->nb_absolute(o); - assert(_Py_CheckSlotResult(o, "__abs__", res != NULL)); - return res; - } - - return type_error("bad operand type for abs(): '%.200s'", o); -} +#define UNARY_FUNC(func, op, meth_name, descr) \ + PyObject * \ + func(PyObject *o) { \ + if (o == NULL) { \ + return null_error(); \ + } \ + \ + PyNumberMethods *m = Py_TYPE(o)->tp_as_number; \ + if (m && m->op) { \ + PyObject *res = (*m->op)(o); \ + assert(_Py_CheckSlotResult(o, #meth_name, res != NULL)); \ + return res; \ + } \ + \ + return type_error("bad operand type for "descr": '%.200s'", o); \ + } + +UNARY_FUNC(PyNumber_Negative, nb_negative, __neg__, "unary -") +UNARY_FUNC(PyNumber_Positive, nb_positive, __pow__, "unary +") +UNARY_FUNC(PyNumber_Invert, nb_invert, __invert__, "unary ~") +UNARY_FUNC(PyNumber_Absolute, nb_absolute, __abs__, "abs()") int diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 67073190cc889d..659de7d3dd5a99 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2007,7 +2007,10 @@ static PyObject * bytearray_join(PyByteArrayObject *self, PyObject *iterable_of_bytes) /*[clinic end generated code: output=a8516370bf68ae08 input=aba6b1f9b30fcb8e]*/ { - return stringlib_bytes_join((PyObject*)self, iterable_of_bytes); + self->ob_exports++; // this protects `self` from being cleared/resized if `iterable_of_bytes` is a custom iterator + PyObject* ret = stringlib_bytes_join((PyObject*)self, iterable_of_bytes); + self->ob_exports--; // unexport `self` + return ret; } /*[clinic input] diff --git a/Objects/classobject.c b/Objects/classobject.c index 618d88894debbe..d7e520f556d9a0 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -319,6 +319,13 @@ method_traverse(PyMethodObject *im, visitproc visit, void *arg) return 0; } +static PyObject * +method_descr_get(PyObject *meth, PyObject *obj, PyObject *cls) +{ + Py_INCREF(meth); + return meth; +} + PyTypeObject PyMethod_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "method", @@ -339,6 +346,7 @@ PyTypeObject PyMethod_Type = { .tp_methods = method_methods, .tp_members = method_memberlist, .tp_getset = method_getset, + .tp_descr_get = method_descr_get, .tp_new = method_new, }; diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 7711434f17c2bc..3e5167d9242fe4 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -954,9 +954,11 @@ PyDoc_STRVAR(unicode_split__doc__, " character (including \\n \\r \\t \\f and spaces) and will discard\n" " empty strings from the result.\n" " maxsplit\n" -" Maximum number of splits (starting from the left).\n" +" Maximum number of splits.\n" " -1 (the default value) means no limit.\n" "\n" +"Splitting starts at the front of the string and works to the end.\n" +"\n" "Note, str.split() is mainly useful for data that has been intentionally\n" "delimited. With natural text that includes punctuation, consider using\n" "the regular expression module."); @@ -1078,7 +1080,7 @@ PyDoc_STRVAR(unicode_rsplit__doc__, " character (including \\n \\r \\t \\f and spaces) and will discard\n" " empty strings from the result.\n" " maxsplit\n" -" Maximum number of splits (starting from the left).\n" +" Maximum number of splits.\n" " -1 (the default value) means no limit.\n" "\n" "Splitting starts at the end of the string and works to the front."); @@ -1505,4 +1507,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=873d8b3d09af3095 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1aab29bab5201c78 input=a9049054013a1b77]*/ diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 57921b110591e5..8d771adf307dc4 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -19,8 +19,9 @@ class property "propertyobject *" "&PyProperty_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=556352653fd4c02e]*/ static void -descr_dealloc(PyDescrObject *descr) +descr_dealloc(PyObject *self) { + PyDescrObject *descr = (PyDescrObject *)self; _PyObject_GC_UNTRACK(descr); Py_XDECREF(descr->d_type); Py_XDECREF(descr->d_name); @@ -47,28 +48,28 @@ descr_repr(PyDescrObject *descr, const char *format) } static PyObject * -method_repr(PyMethodDescrObject *descr) +method_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); } static PyObject * -member_repr(PyMemberDescrObject *descr) +member_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); } static PyObject * -getset_repr(PyGetSetDescrObject *descr) +getset_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); } static PyObject * -wrapperdescr_repr(PyWrapperDescrObject *descr) +wrapperdescr_repr(PyObject *descr) { return descr_repr((PyDescrObject *)descr, ""); @@ -90,8 +91,9 @@ descr_check(PyDescrObject *descr, PyObject *obj) } static PyObject * -classmethod_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) +classmethod_get(PyObject *self, PyObject *obj, PyObject *type) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)self; /* Ensure a valid type. Class methods ignore obj. */ if (type == NULL) { if (obj != NULL) @@ -132,8 +134,9 @@ classmethod_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) +method_get(PyObject *self, PyObject *obj, PyObject *type) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -156,8 +159,9 @@ method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type) +member_get(PyObject *self, PyObject *obj, PyObject *type) { + PyMemberDescrObject *descr = (PyMemberDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -176,8 +180,9 @@ member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -getset_get(PyGetSetDescrObject *descr, PyObject *obj, PyObject *type) +getset_get(PyObject *self, PyObject *obj, PyObject *type) { + PyGetSetDescrObject *descr = (PyGetSetDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -195,8 +200,9 @@ getset_get(PyGetSetDescrObject *descr, PyObject *obj, PyObject *type) } static PyObject * -wrapperdescr_get(PyWrapperDescrObject *descr, PyObject *obj, PyObject *type) +wrapperdescr_get(PyObject *self, PyObject *obj, PyObject *type) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)self; if (obj == NULL) { return Py_NewRef(descr); } @@ -223,8 +229,9 @@ descr_setcheck(PyDescrObject *descr, PyObject *obj, PyObject *value) } static int -member_set(PyMemberDescrObject *descr, PyObject *obj, PyObject *value) +member_set(PyObject *self, PyObject *obj, PyObject *value) { + PyMemberDescrObject *descr = (PyMemberDescrObject *)self; if (descr_setcheck((PyDescrObject *)descr, obj, value) < 0) { return -1; } @@ -232,8 +239,9 @@ member_set(PyMemberDescrObject *descr, PyObject *obj, PyObject *value) } static int -getset_set(PyGetSetDescrObject *descr, PyObject *obj, PyObject *value) +getset_set(PyObject *self, PyObject *obj, PyObject *value) { + PyGetSetDescrObject *descr = (PyGetSetDescrObject *)self; if (descr_setcheck((PyDescrObject *)descr, obj, value) < 0) { return -1; } @@ -479,9 +487,10 @@ method_vectorcall_O( we implement this simply by calling __get__ and then calling the result. */ static PyObject * -classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, +classmethoddescr_call(PyObject *_descr, PyObject *args, PyObject *kwds) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)_descr; Py_ssize_t argc = PyTuple_GET_SIZE(args); if (argc < 1) { PyErr_Format(PyExc_TypeError, @@ -492,7 +501,7 @@ classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, return NULL; } PyObject *self = PyTuple_GET_ITEM(args, 0); - PyObject *bound = classmethod_get(descr, NULL, self); + PyObject *bound = classmethod_get((PyObject *)descr, NULL, self); if (bound == NULL) { return NULL; } @@ -523,8 +532,9 @@ wrapperdescr_raw_call(PyWrapperDescrObject *descr, PyObject *self, } static PyObject * -wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) +wrapperdescr_call(PyObject *_descr, PyObject *args, PyObject *kwds) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)_descr; Py_ssize_t argc; PyObject *self, *result; @@ -563,14 +573,16 @@ wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) static PyObject * -method_get_doc(PyMethodDescrObject *descr, void *closure) +method_get_doc(PyObject *_descr, void *closure) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)_descr; return _PyType_GetDocFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc); } static PyObject * -method_get_text_signature(PyMethodDescrObject *descr, void *closure) +method_get_text_signature(PyObject *_descr, void *closure) { + PyMethodDescrObject *descr = (PyMethodDescrObject *)_descr; return _PyType_GetTextSignatureFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc, descr->d_method->ml_flags); @@ -605,22 +617,24 @@ calculate_qualname(PyDescrObject *descr) } static PyObject * -descr_get_qualname(PyDescrObject *descr, void *Py_UNUSED(ignored)) +descr_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { + PyDescrObject *descr = (PyDescrObject *)self; if (descr->d_qualname == NULL) descr->d_qualname = calculate_qualname(descr); return Py_XNewRef(descr->d_qualname); } static PyObject * -descr_reduce(PyDescrObject *descr, PyObject *Py_UNUSED(ignored)) +descr_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PyDescrObject *descr = (PyDescrObject *)self; return Py_BuildValue("N(OO)", _PyEval_GetBuiltin(&_Py_ID(getattr)), PyDescr_TYPE(descr), PyDescr_NAME(descr)); } static PyMethodDef descr_methods[] = { - {"__reduce__", (PyCFunction)descr_reduce, METH_NOARGS, NULL}, + {"__reduce__", descr_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; @@ -631,15 +645,16 @@ static PyMemberDef descr_members[] = { }; static PyGetSetDef method_getset[] = { - {"__doc__", (getter)method_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, - {"__text_signature__", (getter)method_get_text_signature}, + {"__doc__", method_get_doc}, + {"__qualname__", descr_get_qualname}, + {"__text_signature__", method_get_text_signature}, {0} }; static PyObject * -member_get_doc(PyMemberDescrObject *descr, void *closure) +member_get_doc(PyObject *_descr, void *closure) { + PyMemberDescrObject *descr = (PyMemberDescrObject *)_descr; if (descr->d_member->doc == NULL) { Py_RETURN_NONE; } @@ -647,14 +662,15 @@ member_get_doc(PyMemberDescrObject *descr, void *closure) } static PyGetSetDef member_getset[] = { - {"__doc__", (getter)member_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, + {"__doc__", member_get_doc}, + {"__qualname__", descr_get_qualname}, {0} }; static PyObject * -getset_get_doc(PyGetSetDescrObject *descr, void *closure) +getset_get_doc(PyObject *self, void *closure) { + PyGetSetDescrObject *descr = (PyGetSetDescrObject *)self; if (descr->d_getset->doc == NULL) { Py_RETURN_NONE; } @@ -662,28 +678,30 @@ getset_get_doc(PyGetSetDescrObject *descr, void *closure) } static PyGetSetDef getset_getset[] = { - {"__doc__", (getter)getset_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, + {"__doc__", getset_get_doc}, + {"__qualname__", descr_get_qualname}, {0} }; static PyObject * -wrapperdescr_get_doc(PyWrapperDescrObject *descr, void *closure) +wrapperdescr_get_doc(PyObject *self, void *closure) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)self; return _PyType_GetDocFromInternalDoc(descr->d_base->name, descr->d_base->doc); } static PyObject * -wrapperdescr_get_text_signature(PyWrapperDescrObject *descr, void *closure) +wrapperdescr_get_text_signature(PyObject *self, void *closure) { + PyWrapperDescrObject *descr = (PyWrapperDescrObject *)self; return _PyType_GetTextSignatureFromInternalDoc(descr->d_base->name, descr->d_base->doc, 0); } static PyGetSetDef wrapperdescr_getset[] = { - {"__doc__", (getter)wrapperdescr_get_doc}, - {"__qualname__", (getter)descr_get_qualname}, - {"__text_signature__", (getter)wrapperdescr_get_text_signature}, + {"__doc__", wrapperdescr_get_doc}, + {"__qualname__", descr_get_qualname}, + {"__text_signature__", wrapperdescr_get_text_signature}, {0} }; @@ -700,12 +718,12 @@ PyTypeObject PyMethodDescr_Type = { "method_descriptor", sizeof(PyMethodDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ offsetof(PyMethodDescrObject, vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)method_repr, /* tp_repr */ + method_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -730,7 +748,7 @@ PyTypeObject PyMethodDescr_Type = { method_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)method_get, /* tp_descr_get */ + method_get, /* tp_descr_get */ 0, /* tp_descr_set */ }; @@ -740,17 +758,17 @@ PyTypeObject PyClassMethodDescr_Type = { "classmethod_descriptor", sizeof(PyMethodDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)method_repr, /* tp_repr */ + method_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)classmethoddescr_call, /* tp_call */ + classmethoddescr_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -768,7 +786,7 @@ PyTypeObject PyClassMethodDescr_Type = { method_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)classmethod_get, /* tp_descr_get */ + classmethod_get, /* tp_descr_get */ 0, /* tp_descr_set */ }; @@ -777,12 +795,12 @@ PyTypeObject PyMemberDescr_Type = { "member_descriptor", sizeof(PyMemberDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)member_repr, /* tp_repr */ + member_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -805,8 +823,8 @@ PyTypeObject PyMemberDescr_Type = { member_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)member_get, /* tp_descr_get */ - (descrsetfunc)member_set, /* tp_descr_set */ + member_get, /* tp_descr_get */ + member_set, /* tp_descr_set */ }; PyTypeObject PyGetSetDescr_Type = { @@ -814,12 +832,12 @@ PyTypeObject PyGetSetDescr_Type = { "getset_descriptor", sizeof(PyGetSetDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)getset_repr, /* tp_repr */ + getset_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -842,8 +860,8 @@ PyTypeObject PyGetSetDescr_Type = { getset_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)getset_get, /* tp_descr_get */ - (descrsetfunc)getset_set, /* tp_descr_set */ + getset_get, /* tp_descr_get */ + getset_set, /* tp_descr_set */ }; PyTypeObject PyWrapperDescr_Type = { @@ -851,17 +869,17 @@ PyTypeObject PyWrapperDescr_Type = { "wrapper_descriptor", sizeof(PyWrapperDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + descr_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)wrapperdescr_repr, /* tp_repr */ + wrapperdescr_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)wrapperdescr_call, /* tp_call */ + wrapperdescr_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -880,7 +898,7 @@ PyTypeObject PyWrapperDescr_Type = { wrapperdescr_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ - (descrgetfunc)wrapperdescr_get, /* tp_descr_get */ + wrapperdescr_get, /* tp_descr_get */ 0, /* tp_descr_set */ }; @@ -1022,20 +1040,22 @@ typedef struct { } mappingproxyobject; static Py_ssize_t -mappingproxy_len(mappingproxyobject *pp) +mappingproxy_len(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_Size(pp->mapping); } static PyObject * -mappingproxy_getitem(mappingproxyobject *pp, PyObject *key) +mappingproxy_getitem(PyObject *self, PyObject *key) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_GetItem(pp->mapping, key); } static PyMappingMethods mappingproxy_as_mapping = { - (lenfunc)mappingproxy_len, /* mp_length */ - (binaryfunc)mappingproxy_getitem, /* mp_subscript */ + mappingproxy_len, /* mp_length */ + mappingproxy_getitem, /* mp_subscript */ 0, /* mp_ass_subscript */ }; @@ -1064,8 +1084,9 @@ static PyNumberMethods mappingproxy_as_number = { }; static int -mappingproxy_contains(mappingproxyobject *pp, PyObject *key) +mappingproxy_contains(PyObject *self, PyObject *key) { + mappingproxyobject *pp = (mappingproxyobject *)self; if (PyDict_CheckExact(pp->mapping)) return PyDict_Contains(pp->mapping, key); else @@ -1080,14 +1101,15 @@ static PySequenceMethods mappingproxy_as_sequence = { 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)mappingproxy_contains, /* sq_contains */ + mappingproxy_contains, /* sq_contains */ 0, /* sq_inplace_concat */ 0, /* sq_inplace_repeat */ }; static PyObject * -mappingproxy_get(mappingproxyobject *pp, PyObject *const *args, Py_ssize_t nargs) +mappingproxy_get(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { + mappingproxyobject *pp = (mappingproxyobject *)self; /* newargs: mapping, key, default=None */ PyObject *newargs[3]; newargs[0] = pp->mapping; @@ -1104,32 +1126,37 @@ mappingproxy_get(mappingproxyobject *pp, PyObject *const *args, Py_ssize_t nargs } static PyObject * -mappingproxy_keys(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(keys)); } static PyObject * -mappingproxy_values(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_values(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(values)); } static PyObject * -mappingproxy_items(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(items)); } static PyObject * -mappingproxy_copy(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(copy)); } static PyObject * -mappingproxy_reversed(mappingproxyobject *pp, PyObject *Py_UNUSED(ignored)) +mappingproxy_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_CallMethodNoArgs(pp->mapping, &_Py_ID(__reversed__)); } @@ -1140,50 +1167,55 @@ static PyMethodDef mappingproxy_methods[] = { {"get", _PyCFunction_CAST(mappingproxy_get), METH_FASTCALL, PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." " d defaults to None.")}, - {"keys", (PyCFunction)mappingproxy_keys, METH_NOARGS, + {"keys", mappingproxy_keys, METH_NOARGS, PyDoc_STR("D.keys() -> a set-like object providing a view on D's keys")}, - {"values", (PyCFunction)mappingproxy_values, METH_NOARGS, + {"values", mappingproxy_values, METH_NOARGS, PyDoc_STR("D.values() -> an object providing a view on D's values")}, - {"items", (PyCFunction)mappingproxy_items, METH_NOARGS, + {"items", mappingproxy_items, METH_NOARGS, PyDoc_STR("D.items() -> a set-like object providing a view on D's items")}, - {"copy", (PyCFunction)mappingproxy_copy, METH_NOARGS, + {"copy", mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, - {"__reversed__", (PyCFunction)mappingproxy_reversed, METH_NOARGS, + {"__reversed__", mappingproxy_reversed, METH_NOARGS, PyDoc_STR("D.__reversed__() -> reverse iterator")}, {0} }; static void -mappingproxy_dealloc(mappingproxyobject *pp) +mappingproxy_dealloc(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; _PyObject_GC_UNTRACK(pp); Py_DECREF(pp->mapping); PyObject_GC_Del(pp); } static PyObject * -mappingproxy_getiter(mappingproxyobject *pp) +mappingproxy_getiter(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_GetIter(pp->mapping); } static Py_hash_t -mappingproxy_hash(mappingproxyobject *pp) +mappingproxy_hash(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_Hash(pp->mapping); } static PyObject * -mappingproxy_str(mappingproxyobject *pp) +mappingproxy_str(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyObject_Str(pp->mapping); } static PyObject * -mappingproxy_repr(mappingproxyobject *pp) +mappingproxy_repr(PyObject *self) { + mappingproxyobject *pp = (mappingproxyobject *)self; return PyUnicode_FromFormat("mappingproxy(%R)", pp->mapping); } @@ -1196,8 +1228,9 @@ mappingproxy_traverse(PyObject *self, visitproc visit, void *arg) } static PyObject * -mappingproxy_richcompare(mappingproxyobject *v, PyObject *w, int op) +mappingproxy_richcompare(PyObject *self, PyObject *w, int op) { + mappingproxyobject *v = (mappingproxyobject *)self; return PyObject_RichCompare(v->mapping, w, op); } @@ -1271,8 +1304,9 @@ typedef struct { #define Wrapper_Check(v) Py_IS_TYPE(v, &_PyMethodWrapper_Type) static void -wrapper_dealloc(wrapperobject *wp) +wrapper_dealloc(PyObject *self) { + wrapperobject *wp = (wrapperobject *)self; PyObject_GC_UnTrack(wp); Py_TRASHCAN_BEGIN(wp, wrapper_dealloc) Py_XDECREF(wp->descr); @@ -1308,8 +1342,9 @@ wrapper_richcompare(PyObject *a, PyObject *b, int op) } static Py_hash_t -wrapper_hash(wrapperobject *wp) +wrapper_hash(PyObject *self) { + wrapperobject *wp = (wrapperobject *)self; Py_hash_t x, y; x = _Py_HashPointer(wp->self); y = _Py_HashPointer(wp->descr); @@ -1320,8 +1355,9 @@ wrapper_hash(wrapperobject *wp) } static PyObject * -wrapper_repr(wrapperobject *wp) +wrapper_repr(PyObject *self) { + wrapperobject *wp = (wrapperobject *)self; return PyUnicode_FromFormat("", wp->descr->d_base->name, Py_TYPE(wp->self)->tp_name, @@ -1329,14 +1365,15 @@ wrapper_repr(wrapperobject *wp) } static PyObject * -wrapper_reduce(wrapperobject *wp, PyObject *Py_UNUSED(ignored)) +wrapper_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + wrapperobject *wp = (wrapperobject *)self; return Py_BuildValue("N(OO)", _PyEval_GetBuiltin(&_Py_ID(getattr)), wp->self, PyDescr_NAME(wp->descr)); } static PyMethodDef wrapper_methods[] = { - {"__reduce__", (PyCFunction)wrapper_reduce, METH_NOARGS, NULL}, + {"__reduce__", wrapper_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; @@ -1346,52 +1383,56 @@ static PyMemberDef wrapper_members[] = { }; static PyObject * -wrapper_objclass(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_objclass(PyObject *wp, void *Py_UNUSED(ignored)) { - PyObject *c = (PyObject *)PyDescr_TYPE(wp->descr); + PyObject *c = (PyObject *)PyDescr_TYPE(((wrapperobject *)wp)->descr); return Py_NewRef(c); } static PyObject * -wrapper_name(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_name(PyObject *wp, void *Py_UNUSED(ignored)) { - const char *s = wp->descr->d_base->name; + const char *s = ((wrapperobject *)wp)->descr->d_base->name; return PyUnicode_FromString(s); } static PyObject * -wrapper_doc(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_doc(PyObject *self, void *Py_UNUSED(ignored)) { + wrapperobject *wp = (wrapperobject *)self; return _PyType_GetDocFromInternalDoc(wp->descr->d_base->name, wp->descr->d_base->doc); } static PyObject * -wrapper_text_signature(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_text_signature(PyObject *self, void *Py_UNUSED(ignored)) { + wrapperobject *wp = (wrapperobject *)self; return _PyType_GetTextSignatureFromInternalDoc(wp->descr->d_base->name, wp->descr->d_base->doc, 0); } static PyObject * -wrapper_qualname(wrapperobject *wp, void *Py_UNUSED(ignored)) +wrapper_qualname(PyObject *self, void *Py_UNUSED(ignored)) { - return descr_get_qualname((PyDescrObject *)wp->descr, NULL); + wrapperobject *wp = (wrapperobject *)self; + return descr_get_qualname((PyObject *)wp->descr, NULL); } static PyGetSetDef wrapper_getsets[] = { - {"__objclass__", (getter)wrapper_objclass}, - {"__name__", (getter)wrapper_name}, - {"__qualname__", (getter)wrapper_qualname}, - {"__doc__", (getter)wrapper_doc}, - {"__text_signature__", (getter)wrapper_text_signature}, + {"__objclass__", wrapper_objclass}, + {"__name__", wrapper_name}, + {"__qualname__", wrapper_qualname}, + {"__doc__", wrapper_doc}, + {"__text_signature__", wrapper_text_signature}, {0} }; static PyObject * -wrapper_call(wrapperobject *wp, PyObject *args, PyObject *kwds) +wrapper_call(PyObject *self, PyObject *args, PyObject *kwds) { + wrapperobject *wp = (wrapperobject *)self; return wrapperdescr_raw_call(wp->descr, wp->self, args, kwds); } @@ -1410,17 +1451,17 @@ PyTypeObject _PyMethodWrapper_Type = { sizeof(wrapperobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)wrapper_dealloc, /* tp_dealloc */ + wrapper_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)wrapper_repr, /* tp_repr */ + wrapper_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)wrapper_hash, /* tp_hash */ - (ternaryfunc)wrapper_call, /* tp_call */ + wrapper_hash, /* tp_hash */ + wrapper_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1910,18 +1951,18 @@ PyTypeObject PyDictProxy_Type = { sizeof(mappingproxyobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)mappingproxy_dealloc, /* tp_dealloc */ + mappingproxy_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)mappingproxy_repr, /* tp_repr */ + mappingproxy_repr, /* tp_repr */ &mappingproxy_as_number, /* tp_as_number */ &mappingproxy_as_sequence, /* tp_as_sequence */ &mappingproxy_as_mapping, /* tp_as_mapping */ - (hashfunc)mappingproxy_hash, /* tp_hash */ + mappingproxy_hash, /* tp_hash */ 0, /* tp_call */ - (reprfunc)mappingproxy_str, /* tp_str */ + mappingproxy_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ @@ -1930,9 +1971,9 @@ PyTypeObject PyDictProxy_Type = { 0, /* tp_doc */ mappingproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ - (richcmpfunc)mappingproxy_richcompare, /* tp_richcompare */ + mappingproxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)mappingproxy_getiter, /* tp_iter */ + mappingproxy_getiter, /* tp_iter */ 0, /* tp_iternext */ mappingproxy_methods, /* tp_methods */ 0, /* tp_members */ @@ -1972,7 +2013,7 @@ PyTypeObject PyProperty_Type = { Py_TPFLAGS_BASETYPE, /* tp_flags */ property_init__doc__, /* tp_doc */ property_traverse, /* tp_traverse */ - (inquiry)property_clear, /* tp_clear */ + property_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 70f424e07ece9a..2482a918ba983b 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -237,7 +237,7 @@ equally good collision statistics, needed less code & used less memory. static int dictresize(PyInterpreterState *interp, PyDictObject *mp, uint8_t log_newsize, int unicode); -static PyObject* dict_iter(PyDictObject *dict); +static PyObject* dict_iter(PyObject *dict); #include "clinic/dictobject.c.h" @@ -792,7 +792,7 @@ static PyDictKeysObject * clone_combined_dict_keys(PyDictObject *orig) { assert(PyDict_Check(orig)); - assert(Py_TYPE(orig)->tp_iter == (getiterfunc)dict_iter); + assert(Py_TYPE(orig)->tp_iter == dict_iter); assert(orig->ma_values == NULL); assert(orig->ma_keys != Py_EMPTY_KEYS); assert(orig->ma_keys->dk_refcnt == 1); @@ -2450,8 +2450,9 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) /* Methods */ static void -dict_dealloc(PyDictObject *mp) +dict_dealloc(PyObject *self) { + PyDictObject *mp = (PyDictObject *)self; PyInterpreterState *interp = _PyInterpreterState_GET(); assert(Py_REFCNT(mp) == 0); Py_SET_REFCNT(mp, 1); @@ -2499,8 +2500,9 @@ dict_dealloc(PyDictObject *mp) static PyObject * -dict_repr(PyDictObject *mp) +dict_repr(PyObject *self) { + PyDictObject *mp = (PyDictObject *)self; Py_ssize_t i; PyObject *key = NULL, *value = NULL; _PyUnicodeWriter writer; @@ -2582,14 +2584,16 @@ dict_repr(PyDictObject *mp) } static Py_ssize_t -dict_length(PyDictObject *mp) +dict_length(PyObject *self) { + PyDictObject *mp = (PyDictObject *)self; return mp->ma_used; } static PyObject * -dict_subscript(PyDictObject *mp, PyObject *key) +dict_subscript(PyObject *self, PyObject *key) { + PyDictObject *mp = (PyDictObject *)self; Py_ssize_t ix; Py_hash_t hash; PyObject *value; @@ -2623,18 +2627,18 @@ dict_subscript(PyDictObject *mp, PyObject *key) } static int -dict_ass_sub(PyDictObject *mp, PyObject *v, PyObject *w) +dict_ass_sub(PyObject *mp, PyObject *v, PyObject *w) { if (w == NULL) - return PyDict_DelItem((PyObject *)mp, v); + return PyDict_DelItem(mp, v); else - return PyDict_SetItem((PyObject *)mp, v, w); + return PyDict_SetItem(mp, v, w); } static PyMappingMethods dict_as_mapping = { - (lenfunc)dict_length, /*mp_length*/ - (binaryfunc)dict_subscript, /*mp_subscript*/ - (objobjargproc)dict_ass_sub, /*mp_ass_subscript*/ + dict_length, /*mp_length*/ + dict_subscript, /*mp_subscript*/ + dict_ass_sub, /*mp_ass_subscript*/ }; static PyObject * @@ -2925,7 +2929,7 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) return -1; } mp = (PyDictObject*)a; - if (PyDict_Check(b) && (Py_TYPE(b)->tp_iter == (getiterfunc)dict_iter)) { + if (PyDict_Check(b) && (Py_TYPE(b)->tp_iter == dict_iter)) { other = (PyDictObject*)b; if (other == mp || other->ma_used == 0) /* a.update(a) or a.update({}); nothing to do */ @@ -3105,9 +3109,9 @@ _PyDict_MergeEx(PyObject *a, PyObject *b, int override) } static PyObject * -dict_copy(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) +dict_copy(PyObject *mp, PyObject *Py_UNUSED(ignored)) { - return PyDict_Copy((PyObject*)mp); + return PyDict_Copy(mp); } PyObject * @@ -3155,7 +3159,7 @@ PyDict_Copy(PyObject *o) return (PyObject *)split_copy; } - if (Py_TYPE(mp)->tp_iter == (getiterfunc)dict_iter && + if (Py_TYPE(mp)->tp_iter == dict_iter && mp->ma_values == NULL && (mp->ma_used >= (mp->ma_keys->dk_nentries * 2) / 3)) { @@ -3509,9 +3513,9 @@ dict_setdefault_impl(PyDictObject *self, PyObject *key, } static PyObject * -dict_clear(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) +dict_clear(PyObject *mp, PyObject *Py_UNUSED(ignored)) { - PyDict_Clear((PyObject *)mp); + PyDict_Clear(mp); Py_RETURN_NONE; } @@ -3700,8 +3704,9 @@ _PyDict_KeysSize(PyDictKeysObject *keys) } static PyObject * -dict_sizeof(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) +dict_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PyDictObject *mp = (PyDictObject *)self; return PyLong_FromSsize_t(_PyDict_SizeOf(mp)); } @@ -3763,9 +3768,9 @@ PyDoc_STRVAR(values__doc__, static PyMethodDef mapp_methods[] = { DICT___CONTAINS___METHODDEF - {"__getitem__", _PyCFunction_CAST(dict_subscript), METH_O | METH_COEXIST, + {"__getitem__", dict_subscript, METH_O | METH_COEXIST, getitem__doc__}, - {"__sizeof__", _PyCFunction_CAST(dict_sizeof), METH_NOARGS, + {"__sizeof__", dict_sizeof, METH_NOARGS, sizeof__doc__}, DICT_GET_METHODDEF DICT_SETDEFAULT_METHODDEF @@ -3780,9 +3785,9 @@ static PyMethodDef mapp_methods[] = { {"update", _PyCFunction_CAST(dict_update), METH_VARARGS | METH_KEYWORDS, update__doc__}, DICT_FROMKEYS_METHODDEF - {"clear", (PyCFunction)dict_clear, METH_NOARGS, + {"clear", dict_clear, METH_NOARGS, clear__doc__}, - {"copy", (PyCFunction)dict_copy, METH_NOARGS, + {"copy", dict_copy, METH_NOARGS, copy__doc__}, DICT___REVERSED___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, @@ -3937,8 +3942,9 @@ dict_vectorcall(PyObject *type, PyObject * const*args, } static PyObject * -dict_iter(PyDictObject *dict) +dict_iter(PyObject *self) { + PyDictObject *dict = (PyDictObject *)self; return dictiter_new(dict, &PyDictIterKey_Type); } @@ -3958,12 +3964,12 @@ PyTypeObject PyDict_Type = { "dict", sizeof(PyDictObject), 0, - (destructor)dict_dealloc, /* tp_dealloc */ + dict_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dict_repr, /* tp_repr */ + dict_repr, /* tp_repr */ &dict_as_number, /* tp_as_number */ &dict_as_sequence, /* tp_as_sequence */ &dict_as_mapping, /* tp_as_mapping */ @@ -3981,7 +3987,7 @@ PyTypeObject PyDict_Type = { dict_tp_clear, /* tp_clear */ dict_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dict_iter, /* tp_iter */ + dict_iter, /* tp_iter */ 0, /* tp_iternext */ mapp_methods, /* tp_methods */ 0, /* tp_members */ @@ -4128,8 +4134,9 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) } static void -dictiter_dealloc(dictiterobject *di) +dictiter_dealloc(PyObject *self) { + dictiterobject *di = (dictiterobject *)self; /* bpo-31095: UnTrack is needed before calling any callbacks */ _PyObject_GC_UNTRACK(di); Py_XDECREF(di->di_dict); @@ -4138,16 +4145,18 @@ dictiter_dealloc(dictiterobject *di) } static int -dictiter_traverse(dictiterobject *di, visitproc visit, void *arg) +dictiter_traverse(PyObject *self, visitproc visit, void *arg) { + dictiterobject *di = (dictiterobject *)self; Py_VISIT(di->di_dict); Py_VISIT(di->di_result); return 0; } static PyObject * -dictiter_len(dictiterobject *di, PyObject *Py_UNUSED(ignored)) +dictiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + dictiterobject *di = (dictiterobject *)self; Py_ssize_t len = 0; if (di->di_dict != NULL && di->di_used == di->di_dict->ma_used) len = di->len; @@ -4158,21 +4167,22 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -dictiter_reduce(dictiterobject *di, PyObject *Py_UNUSED(ignored)); +dictiter_reduce(PyObject *di, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef dictiter_methods[] = { - {"__length_hint__", _PyCFunction_CAST(dictiter_len), METH_NOARGS, + {"__length_hint__", dictiter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", _PyCFunction_CAST(dictiter_reduce), METH_NOARGS, + {"__reduce__", dictiter_reduce, METH_NOARGS, reduce_doc}, {NULL, NULL} /* sentinel */ }; static PyObject* -dictiter_iternextkey(dictiterobject *di) +dictiter_iternextkey(PyObject *self) { + dictiterobject *di = (dictiterobject *)self; PyObject *key; Py_ssize_t i; PyDictKeysObject *k; @@ -4244,7 +4254,7 @@ PyTypeObject PyDictIterKey_Type = { sizeof(dictiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictiter_dealloc, /* tp_dealloc */ + dictiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4261,19 +4271,20 @@ PyTypeObject PyDictIterKey_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictiter_traverse, /* tp_traverse */ + dictiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)dictiter_iternextkey, /* tp_iternext */ + dictiter_iternextkey, /* tp_iternext */ dictiter_methods, /* tp_methods */ 0, }; static PyObject * -dictiter_iternextvalue(dictiterobject *di) +dictiter_iternextvalue(PyObject *self) { + dictiterobject *di = (dictiterobject *)self; PyObject *value; Py_ssize_t i; PyDictObject *d = di->di_dict; @@ -4343,7 +4354,7 @@ PyTypeObject PyDictIterValue_Type = { sizeof(dictiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictiter_dealloc, /* tp_dealloc */ + dictiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4360,19 +4371,20 @@ PyTypeObject PyDictIterValue_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictiter_traverse, /* tp_traverse */ + dictiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)dictiter_iternextvalue, /* tp_iternext */ + dictiter_iternextvalue, /* tp_iternext */ dictiter_methods, /* tp_methods */ 0, }; static PyObject * -dictiter_iternextitem(dictiterobject *di) +dictiter_iternextitem(PyObject *self) { + dictiterobject *di = (dictiterobject *)self; PyObject *key, *value, *result; Py_ssize_t i; PyDictObject *d = di->di_dict; @@ -4467,7 +4479,7 @@ PyTypeObject PyDictIterItem_Type = { sizeof(dictiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictiter_dealloc, /* tp_dealloc */ + dictiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4484,12 +4496,12 @@ PyTypeObject PyDictIterItem_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictiter_traverse, /* tp_traverse */ + dictiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)dictiter_iternextitem, /* tp_iternext */ + dictiter_iternextitem, /* tp_iternext */ dictiter_methods, /* tp_methods */ 0, }; @@ -4498,8 +4510,9 @@ PyTypeObject PyDictIterItem_Type = { /* dictreviter */ static PyObject * -dictreviter_iternext(dictiterobject *di) +dictreviter_iternext(PyObject *self) { + dictiterobject *di = (dictiterobject *)self; PyDictObject *d = di->di_dict; if (d == NULL) { @@ -4600,11 +4613,11 @@ PyTypeObject PyDictRevIterKey_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reversekeyiterator", sizeof(dictiterobject), - .tp_dealloc = (destructor)dictiter_dealloc, + .tp_dealloc = dictiter_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)dictiter_traverse, + .tp_traverse = dictiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)dictreviter_iternext, + .tp_iternext = dictreviter_iternext, .tp_methods = dictiter_methods }; @@ -4624,8 +4637,9 @@ dict___reversed___impl(PyDictObject *self) } static PyObject * -dictiter_reduce(dictiterobject *di, PyObject *Py_UNUSED(ignored)) +dictiter_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + dictiterobject *di = (dictiterobject *)self; /* copy the iterator state */ dictiterobject tmp = *di; Py_XINCREF(tmp.di_dict); @@ -4641,11 +4655,11 @@ PyTypeObject PyDictRevIterItem_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reverseitemiterator", sizeof(dictiterobject), - .tp_dealloc = (destructor)dictiter_dealloc, + .tp_dealloc = dictiter_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)dictiter_traverse, + .tp_traverse = dictiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)dictreviter_iternext, + .tp_iternext = dictreviter_iternext, .tp_methods = dictiter_methods }; @@ -4653,11 +4667,11 @@ PyTypeObject PyDictRevIterValue_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_reversevalueiterator", sizeof(dictiterobject), - .tp_dealloc = (destructor)dictiter_dealloc, + .tp_dealloc = dictiter_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)dictiter_traverse, + .tp_traverse = dictiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)dictreviter_iternext, + .tp_iternext = dictreviter_iternext, .tp_methods = dictiter_methods }; @@ -4668,8 +4682,9 @@ PyTypeObject PyDictRevIterValue_Type = { /* The instance lay-out is the same for all three; but the type differs. */ static void -dictview_dealloc(_PyDictViewObject *dv) +dictview_dealloc(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; /* bpo-31095: UnTrack is needed before calling any callbacks */ _PyObject_GC_UNTRACK(dv); Py_XDECREF(dv->dv_dict); @@ -4677,15 +4692,17 @@ dictview_dealloc(_PyDictViewObject *dv) } static int -dictview_traverse(_PyDictViewObject *dv, visitproc visit, void *arg) +dictview_traverse(PyObject *self, visitproc visit, void *arg) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_VISIT(dv->dv_dict); return 0; } static Py_ssize_t -dictview_len(_PyDictViewObject *dv) +dictview_len(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_ssize_t len = 0; if (dv->dv_dict != NULL) len = dv->dv_dict->ma_used; @@ -4825,8 +4842,9 @@ dictview_richcompare(PyObject *self, PyObject *other, int op) } static PyObject * -dictview_repr(_PyDictViewObject *dv) +dictview_repr(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; PyObject *seq; PyObject *result = NULL; Py_ssize_t rc; @@ -4850,8 +4868,9 @@ dictview_repr(_PyDictViewObject *dv) /*** dict_keys ***/ static PyObject * -dictkeys_iter(_PyDictViewObject *dv) +dictkeys_iter(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -4859,22 +4878,23 @@ dictkeys_iter(_PyDictViewObject *dv) } static int -dictkeys_contains(_PyDictViewObject *dv, PyObject *obj) +dictkeys_contains(PyObject *self, PyObject *obj) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) return 0; return PyDict_Contains((PyObject *)dv->dv_dict, obj); } static PySequenceMethods dictkeys_as_sequence = { - (lenfunc)dictview_len, /* sq_length */ + dictview_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)dictkeys_contains, /* sq_contains */ + dictkeys_contains, /* sq_contains */ }; // Create a set object from dictviews object. @@ -4914,7 +4934,7 @@ dictviews_sub(PyObject *self, PyObject *other) } static int -dictitems_contains(_PyDictViewObject *dv, PyObject *obj); +dictitems_contains(PyObject *dv, PyObject *obj); PyObject * _PyDictView_Intersect(PyObject* self, PyObject *other) @@ -4924,7 +4944,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) PyObject *key; Py_ssize_t len_self; int rv; - int (*dict_contains)(_PyDictViewObject *, PyObject *); + objobjproc dict_contains; /* Python interpreter swaps parameters when dict view is on right side of & */ @@ -4934,7 +4954,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) self = tmp; } - len_self = dictview_len((_PyDictViewObject *)self); + len_self = dictview_len(self); /* if other is a set and self is smaller than other, reuse set intersection logic */ @@ -4946,7 +4966,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) /* if other is another dict view, and it is bigger than self, swap them */ if (PyDictViewSet_Check(other)) { - Py_ssize_t len_other = dictview_len((_PyDictViewObject *)other); + Py_ssize_t len_other = dictview_len(other); if (len_other > len_self) { PyObject *tmp = other; other = self; @@ -4976,7 +4996,7 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) } while ((key = PyIter_Next(it)) != NULL) { - rv = dict_contains((_PyDictViewObject *)self, key); + rv = dict_contains(self, key); if (rv < 0) { goto error; } @@ -5150,7 +5170,7 @@ dictviews_isdisjoint(PyObject *self, PyObject *other) PyObject *item = NULL; if (self == other) { - if (dictview_len((_PyDictViewObject *)self) == 0) + if (dictview_len(self) == 0) Py_RETURN_TRUE; else Py_RETURN_FALSE; @@ -5159,7 +5179,7 @@ dictviews_isdisjoint(PyObject *self, PyObject *other) /* Iterate over the shorter object (only if other is a set, * because PySequence_Contains may be expensive otherwise): */ if (PyAnySet_Check(other) || PyDictViewSet_Check(other)) { - Py_ssize_t len_self = dictview_len((_PyDictViewObject *)self); + Py_ssize_t len_self = dictview_len(self); Py_ssize_t len_other = PyObject_Size(other); if (len_other == -1) return NULL; @@ -5197,15 +5217,15 @@ dictviews_isdisjoint(PyObject *self, PyObject *other) PyDoc_STRVAR(isdisjoint_doc, "Return True if the view and the given iterable have a null intersection."); -static PyObject* dictkeys_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)); +static PyObject* dictkeys_reversed(PyObject *dv, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reversed_keys_doc, "Return a reverse iterator over the dict keys."); static PyMethodDef dictkeys_methods[] = { - {"isdisjoint", (PyCFunction)dictviews_isdisjoint, METH_O, + {"isdisjoint", dictviews_isdisjoint, METH_O, isdisjoint_doc}, - {"__reversed__", _PyCFunction_CAST(dictkeys_reversed), METH_NOARGS, + {"__reversed__", dictkeys_reversed, METH_NOARGS, reversed_keys_doc}, {NULL, NULL} /* sentinel */ }; @@ -5216,12 +5236,12 @@ PyTypeObject PyDictKeys_Type = { sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictview_dealloc, /* tp_dealloc */ + dictview_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dictview_repr, /* tp_repr */ + dictview_repr, /* tp_repr */ &dictviews_as_number, /* tp_as_number */ &dictkeys_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -5233,11 +5253,11 @@ PyTypeObject PyDictKeys_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictview_traverse, /* tp_traverse */ + dictview_traverse, /* tp_traverse */ 0, /* tp_clear */ dictview_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dictkeys_iter, /* tp_iter */ + dictkeys_iter, /* tp_iter */ 0, /* tp_iternext */ dictkeys_methods, /* tp_methods */ .tp_getset = dictview_getset, @@ -5250,8 +5270,9 @@ dictkeys_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) } static PyObject * -dictkeys_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) +dictkeys_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5261,8 +5282,9 @@ dictkeys_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) /*** dict_items ***/ static PyObject * -dictitems_iter(_PyDictViewObject *dv) +dictitems_iter(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5270,8 +5292,9 @@ dictitems_iter(_PyDictViewObject *dv) } static int -dictitems_contains(_PyDictViewObject *dv, PyObject *obj) +dictitems_contains(PyObject *self, PyObject *obj) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; int result; PyObject *key, *value, *found; if (dv->dv_dict == NULL) @@ -5289,25 +5312,25 @@ dictitems_contains(_PyDictViewObject *dv, PyObject *obj) } static PySequenceMethods dictitems_as_sequence = { - (lenfunc)dictview_len, /* sq_length */ + dictview_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)dictitems_contains, /* sq_contains */ + dictitems_contains, /* sq_contains */ }; -static PyObject* dictitems_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)); +static PyObject* dictitems_reversed(PyObject *dv, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reversed_items_doc, "Return a reverse iterator over the dict items."); static PyMethodDef dictitems_methods[] = { - {"isdisjoint", (PyCFunction)dictviews_isdisjoint, METH_O, + {"isdisjoint", dictviews_isdisjoint, METH_O, isdisjoint_doc}, - {"__reversed__", (PyCFunction)dictitems_reversed, METH_NOARGS, + {"__reversed__", dictitems_reversed, METH_NOARGS, reversed_items_doc}, {NULL, NULL} /* sentinel */ }; @@ -5318,12 +5341,12 @@ PyTypeObject PyDictItems_Type = { sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictview_dealloc, /* tp_dealloc */ + dictview_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dictview_repr, /* tp_repr */ + dictview_repr, /* tp_repr */ &dictviews_as_number, /* tp_as_number */ &dictitems_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -5335,11 +5358,11 @@ PyTypeObject PyDictItems_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictview_traverse, /* tp_traverse */ + dictview_traverse, /* tp_traverse */ 0, /* tp_clear */ dictview_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dictitems_iter, /* tp_iter */ + dictitems_iter, /* tp_iter */ 0, /* tp_iternext */ dictitems_methods, /* tp_methods */ .tp_getset = dictview_getset, @@ -5352,8 +5375,9 @@ dictitems_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) } static PyObject * -dictitems_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) +dictitems_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5363,8 +5387,9 @@ dictitems_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) /*** dict_values ***/ static PyObject * -dictvalues_iter(_PyDictViewObject *dv) +dictvalues_iter(PyObject *self) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } @@ -5372,7 +5397,7 @@ dictvalues_iter(_PyDictViewObject *dv) } static PySequenceMethods dictvalues_as_sequence = { - (lenfunc)dictview_len, /* sq_length */ + dictview_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ @@ -5382,13 +5407,13 @@ static PySequenceMethods dictvalues_as_sequence = { (objobjproc)0, /* sq_contains */ }; -static PyObject* dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)); +static PyObject* dictvalues_reversed(PyObject *dv, PyObject *Py_UNUSED(ignored)); PyDoc_STRVAR(reversed_values_doc, "Return a reverse iterator over the dict values."); static PyMethodDef dictvalues_methods[] = { - {"__reversed__", (PyCFunction)dictvalues_reversed, METH_NOARGS, + {"__reversed__", dictvalues_reversed, METH_NOARGS, reversed_values_doc}, {NULL, NULL} /* sentinel */ }; @@ -5399,12 +5424,12 @@ PyTypeObject PyDictValues_Type = { sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)dictview_dealloc, /* tp_dealloc */ + dictview_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)dictview_repr, /* tp_repr */ + dictview_repr, /* tp_repr */ 0, /* tp_as_number */ &dictvalues_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -5416,11 +5441,11 @@ PyTypeObject PyDictValues_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)dictview_traverse, /* tp_traverse */ + dictview_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)dictvalues_iter, /* tp_iter */ + dictvalues_iter, /* tp_iter */ 0, /* tp_iternext */ dictvalues_methods, /* tp_methods */ .tp_getset = dictview_getset, @@ -5433,8 +5458,9 @@ dictvalues_new(PyObject *dict, PyObject *Py_UNUSED(ignored)) } static PyObject * -dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) +dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyDictViewObject *dv = (_PyDictViewObject *)self; if (dv->dv_dict == NULL) { Py_RETURN_NONE; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index be330a775872c2..cafe4ef6141d9a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -811,7 +811,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore PyObject *exc = _PyFrame_StackPop(f->f_frame); assert(PyExceptionInstance_Check(exc) || exc == Py_None); PyThreadState *tstate = _PyThreadState_GET(); - Py_XSETREF(tstate->exc_info->exc_value, exc); + Py_XSETREF(tstate->exc_info->exc_value, exc == Py_None ? NULL : exc); } else { PyObject *v = _PyFrame_StackPop(f->f_frame); diff --git a/Objects/genobject.c b/Objects/genobject.c index f98aa357cd2ce1..9614713883741c 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -732,7 +732,7 @@ _gen_getframe(PyGenObject *gen, const char *const name) if (PySys_Audit("object.__getattr__", "Os", gen, name) < 0) { return NULL; } - if (gen->gi_frame_state == FRAME_CLEARED) { + if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { Py_RETURN_NONE; } return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject((_PyInterpreterFrame *)gen->gi_iframe)); diff --git a/Objects/listobject.c b/Objects/listobject.c index 2d04218439bd20..dfb8cd2b106511 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -343,8 +343,9 @@ PyList_Append(PyObject *op, PyObject *newitem) /* Methods */ static void -list_dealloc(PyListObject *op) +list_dealloc(PyObject *self) { + PyListObject *op = (PyListObject *)self; Py_ssize_t i; PyObject_GC_UnTrack(op); Py_TRASHCAN_BEGIN(op, list_dealloc) @@ -378,8 +379,9 @@ list_dealloc(PyListObject *op) } static PyObject * -list_repr(PyListObject *v) +list_repr(PyObject *self) { + PyListObject *v = (PyListObject *)self; Py_ssize_t i; PyObject *s; _PyUnicodeWriter writer; @@ -434,14 +436,15 @@ list_repr(PyListObject *v) } static Py_ssize_t -list_length(PyListObject *a) +list_length(PyObject *a) { return Py_SIZE(a); } static int -list_contains(PyListObject *a, PyObject *el) +list_contains(PyObject *aa, PyObject *el) { + PyListObject *a = (PyListObject *)aa; PyObject *item; Py_ssize_t i; int cmp; @@ -456,8 +459,9 @@ list_contains(PyListObject *a, PyObject *el) } static PyObject * -list_item(PyListObject *a, Py_ssize_t i) +list_item(PyObject *aa, Py_ssize_t i) { + PyListObject *a = (PyListObject *)aa; if (!valid_index(i, Py_SIZE(a))) { PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); return NULL; @@ -512,8 +516,9 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) } static PyObject * -list_concat(PyListObject *a, PyObject *bb) +list_concat(PyObject *aa, PyObject *bb) { + PyListObject *a = (PyListObject *)aa; Py_ssize_t size; Py_ssize_t i; PyObject **src, **dest; @@ -552,8 +557,9 @@ list_concat(PyListObject *a, PyObject *bb) } static PyObject * -list_repeat(PyListObject *a, Py_ssize_t n) +list_repeat(PyObject *aa, Py_ssize_t n) { + PyListObject *a = (PyListObject *)aa; const Py_ssize_t input_size = Py_SIZE(a); if (input_size == 0 || n <= 0) return PyList_New(0); @@ -616,9 +622,9 @@ list_clear(PyListObject *a) } static int -list_clear_slot(PyListObject *self) +list_clear_slot(PyObject *self) { - list_clear(self); + list_clear((PyListObject *)self); return 0; } @@ -745,8 +751,9 @@ PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) } static PyObject * -list_inplace_repeat(PyListObject *self, Py_ssize_t n) +list_inplace_repeat(PyObject *_self, Py_ssize_t n) { + PyListObject *self = (PyListObject *)_self; Py_ssize_t input_size = PyList_GET_SIZE(self); if (input_size == 0 || n == 1) { return Py_NewRef(self); @@ -776,8 +783,9 @@ list_inplace_repeat(PyListObject *self, Py_ssize_t n) } static int -list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v) +list_ass_item(PyObject *aa, Py_ssize_t i, PyObject *v) { + PyListObject *a = (PyListObject *)aa; if (!valid_index(i, Py_SIZE(a))) { PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); @@ -1044,8 +1052,9 @@ PyList_Clear(PyObject *self) static PyObject * -list_inplace_concat(PyListObject *self, PyObject *other) +list_inplace_concat(PyObject *_self, PyObject *other) { + PyListObject *self = (PyListObject *)_self; if (list_extend(self, other) < 0) { return NULL; } @@ -2756,8 +2765,9 @@ list_remove(PyListObject *self, PyObject *value) } static int -list_traverse(PyListObject *o, visitproc visit, void *arg) +list_traverse(PyObject *self, visitproc visit, void *arg) { + PyListObject *o = (PyListObject *)self; Py_ssize_t i; for (i = Py_SIZE(o); --i >= 0; ) @@ -2897,10 +2907,10 @@ list___sizeof___impl(PyListObject *self) } static PyObject *list_iter(PyObject *seq); -static PyObject *list_subscript(PyListObject*, PyObject*); +static PyObject *list_subscript(PyObject*, PyObject*); static PyMethodDef list_methods[] = { - {"__getitem__", (PyCFunction)list_subscript, METH_O|METH_COEXIST, + {"__getitem__", list_subscript, METH_O|METH_COEXIST, PyDoc_STR("__getitem__($self, index, /)\n--\n\nReturn self[index].")}, LIST___REVERSED___METHODDEF LIST___SIZEOF___METHODDEF @@ -2920,21 +2930,22 @@ static PyMethodDef list_methods[] = { }; static PySequenceMethods list_as_sequence = { - (lenfunc)list_length, /* sq_length */ - (binaryfunc)list_concat, /* sq_concat */ - (ssizeargfunc)list_repeat, /* sq_repeat */ - (ssizeargfunc)list_item, /* sq_item */ + list_length, /* sq_length */ + list_concat, /* sq_concat */ + list_repeat, /* sq_repeat */ + list_item, /* sq_item */ 0, /* sq_slice */ - (ssizeobjargproc)list_ass_item, /* sq_ass_item */ + list_ass_item, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)list_contains, /* sq_contains */ - (binaryfunc)list_inplace_concat, /* sq_inplace_concat */ - (ssizeargfunc)list_inplace_repeat, /* sq_inplace_repeat */ + list_contains, /* sq_contains */ + list_inplace_concat, /* sq_inplace_concat */ + list_inplace_repeat, /* sq_inplace_repeat */ }; static PyObject * -list_subscript(PyListObject* self, PyObject* item) +list_subscript(PyObject* _self, PyObject* item) { + PyListObject* self = (PyListObject*)_self; if (_PyIndex_Check(item)) { Py_ssize_t i; i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -2942,7 +2953,7 @@ list_subscript(PyListObject* self, PyObject* item) return NULL; if (i < 0) i += PyList_GET_SIZE(self); - return list_item(self, i); + return list_item((PyObject *)self, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, i; @@ -2987,15 +2998,16 @@ list_subscript(PyListObject* self, PyObject* item) } static int -list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) +list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) { + PyListObject *self = (PyListObject *)_self; if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) i += PyList_GET_SIZE(self); - return list_ass_item(self, i, value); + return list_ass_item((PyObject *)self, i, value); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength; @@ -3149,9 +3161,9 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) } static PyMappingMethods list_as_mapping = { - (lenfunc)list_length, - (binaryfunc)list_subscript, - (objobjargproc)list_ass_subscript + list_length, + list_subscript, + list_ass_subscript }; PyTypeObject PyList_Type = { @@ -3159,12 +3171,12 @@ PyTypeObject PyList_Type = { "list", sizeof(PyListObject), 0, - (destructor)list_dealloc, /* tp_dealloc */ + list_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)list_repr, /* tp_repr */ + list_repr, /* tp_repr */ 0, /* tp_as_number */ &list_as_sequence, /* tp_as_sequence */ &list_as_mapping, /* tp_as_mapping */ @@ -3178,8 +3190,8 @@ PyTypeObject PyList_Type = { Py_TPFLAGS_BASETYPE | Py_TPFLAGS_LIST_SUBCLASS | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_SEQUENCE, /* tp_flags */ list___init____doc__, /* tp_doc */ - (traverseproc)list_traverse, /* tp_traverse */ - (inquiry)list_clear_slot, /* tp_clear */ + list_traverse, /* tp_traverse */ + list_clear_slot, /* tp_clear */ list_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ list_iter, /* tp_iter */ @@ -3201,22 +3213,22 @@ PyTypeObject PyList_Type = { /*********************** List Iterator **************************/ -static void listiter_dealloc(_PyListIterObject *); -static int listiter_traverse(_PyListIterObject *, visitproc, void *); -static PyObject *listiter_next(_PyListIterObject *); -static PyObject *listiter_len(_PyListIterObject *, PyObject *); +static void listiter_dealloc(PyObject *); +static int listiter_traverse(PyObject *, visitproc, void *); +static PyObject *listiter_next(PyObject *); +static PyObject *listiter_len(PyObject *, PyObject *); static PyObject *listiter_reduce_general(void *_it, int forward); -static PyObject *listiter_reduce(_PyListIterObject *, PyObject *); -static PyObject *listiter_setstate(_PyListIterObject *, PyObject *state); +static PyObject *listiter_reduce(PyObject *, PyObject *); +static PyObject *listiter_setstate(PyObject *, PyObject *state); PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef listiter_methods[] = { - {"__length_hint__", (PyCFunction)listiter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)listiter_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)listiter_setstate, METH_O, setstate_doc}, + {"__length_hint__", listiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", listiter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", listiter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -3226,7 +3238,7 @@ PyTypeObject PyListIter_Type = { sizeof(_PyListIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)listiter_dealloc, /* tp_dealloc */ + listiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3243,12 +3255,12 @@ PyTypeObject PyListIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)listiter_traverse, /* tp_traverse */ + listiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)listiter_next, /* tp_iternext */ + listiter_next, /* tp_iternext */ listiter_methods, /* tp_methods */ 0, /* tp_members */ }; @@ -3273,23 +3285,25 @@ list_iter(PyObject *seq) } static void -listiter_dealloc(_PyListIterObject *it) +listiter_dealloc(PyObject *self) { + _PyListIterObject *it = (_PyListIterObject *)self; _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -listiter_traverse(_PyListIterObject *it, visitproc visit, void *arg) +listiter_traverse(PyObject *it, visitproc visit, void *arg) { - Py_VISIT(it->it_seq); + Py_VISIT(((_PyListIterObject *)it)->it_seq); return 0; } static PyObject * -listiter_next(_PyListIterObject *it) +listiter_next(PyObject *self) { + _PyListIterObject *it = (_PyListIterObject *)self; PyListObject *seq; PyObject *item; @@ -3311,8 +3325,9 @@ listiter_next(_PyListIterObject *it) } static PyObject * -listiter_len(_PyListIterObject *it, PyObject *Py_UNUSED(ignored)) +listiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyListIterObject *it = (_PyListIterObject *)self; Py_ssize_t len; if (it->it_seq) { len = PyList_GET_SIZE(it->it_seq) - it->it_index; @@ -3323,14 +3338,15 @@ listiter_len(_PyListIterObject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -listiter_reduce(_PyListIterObject *it, PyObject *Py_UNUSED(ignored)) +listiter_reduce(PyObject *it, PyObject *Py_UNUSED(ignored)) { return listiter_reduce_general(it, 1); } static PyObject * -listiter_setstate(_PyListIterObject *it, PyObject *state) +listiter_setstate(PyObject *self, PyObject *state) { + _PyListIterObject *it = (_PyListIterObject *)self; Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; @@ -3352,17 +3368,17 @@ typedef struct { PyListObject *it_seq; /* Set to NULL when iterator is exhausted */ } listreviterobject; -static void listreviter_dealloc(listreviterobject *); -static int listreviter_traverse(listreviterobject *, visitproc, void *); -static PyObject *listreviter_next(listreviterobject *); -static PyObject *listreviter_len(listreviterobject *, PyObject *); -static PyObject *listreviter_reduce(listreviterobject *, PyObject *); -static PyObject *listreviter_setstate(listreviterobject *, PyObject *); +static void listreviter_dealloc(PyObject *); +static int listreviter_traverse(PyObject *, visitproc, void *); +static PyObject *listreviter_next(PyObject *); +static PyObject *listreviter_len(PyObject *, PyObject *); +static PyObject *listreviter_reduce(PyObject *, PyObject *); +static PyObject *listreviter_setstate(PyObject *, PyObject *); static PyMethodDef listreviter_methods[] = { - {"__length_hint__", (PyCFunction)listreviter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)listreviter_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)listreviter_setstate, METH_O, setstate_doc}, + {"__length_hint__", listreviter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", listreviter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", listreviter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -3372,7 +3388,7 @@ PyTypeObject PyListRevIter_Type = { sizeof(listreviterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)listreviter_dealloc, /* tp_dealloc */ + listreviter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3389,12 +3405,12 @@ PyTypeObject PyListRevIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)listreviter_traverse, /* tp_traverse */ + listreviter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)listreviter_next, /* tp_iternext */ + listreviter_next, /* tp_iternext */ listreviter_methods, /* tp_methods */ 0, }; @@ -3422,23 +3438,25 @@ list___reversed___impl(PyListObject *self) } static void -listreviter_dealloc(listreviterobject *it) +listreviter_dealloc(PyObject *self) { + listreviterobject *it = (listreviterobject *)self; PyObject_GC_UnTrack(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -listreviter_traverse(listreviterobject *it, visitproc visit, void *arg) +listreviter_traverse(PyObject *it, visitproc visit, void *arg) { - Py_VISIT(it->it_seq); + Py_VISIT(((listreviterobject *)it)->it_seq); return 0; } static PyObject * -listreviter_next(listreviterobject *it) +listreviter_next(PyObject *self) { + listreviterobject *it = (listreviterobject *)self; PyObject *item; Py_ssize_t index; PyListObject *seq; @@ -3463,8 +3481,9 @@ listreviter_next(listreviterobject *it) } static PyObject * -listreviter_len(listreviterobject *it, PyObject *Py_UNUSED(ignored)) +listreviter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + listreviterobject *it = (listreviterobject *)self; Py_ssize_t len = it->it_index + 1; if (it->it_seq == NULL || PyList_GET_SIZE(it->it_seq) < len) len = 0; @@ -3472,14 +3491,15 @@ listreviter_len(listreviterobject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -listreviter_reduce(listreviterobject *it, PyObject *Py_UNUSED(ignored)) +listreviter_reduce(PyObject *it, PyObject *Py_UNUSED(ignored)) { return listiter_reduce_general(it, 0); } static PyObject * -listreviter_setstate(listreviterobject *it, PyObject *state) +listreviter_setstate(PyObject *self, PyObject *state) { + listreviterobject *it = (listreviterobject *)self; Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index d45d09d4ab9a50..0f3599340318f0 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -60,7 +60,7 @@ Final form: Iterating over the table. ------------------------- -For the `co_lines` attribute we want to emit the full form, omitting the (350, 360, No line number) and empty entries. +For the `co_lines` method we want to emit the full form, omitting the (350, 360, No line number) and empty entries. The code is as follows: diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index bcdd2ff0ceafe6..6a38952fdc1f3b 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -119,8 +119,9 @@ mbuf_release(_PyManagedBufferObject *self) } static void -mbuf_dealloc(_PyManagedBufferObject *self) +mbuf_dealloc(PyObject *_self) { + _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; assert(self->exports == 0); mbuf_release(self); if (self->flags&_Py_MANAGED_BUFFER_FREE_FORMAT) @@ -129,15 +130,17 @@ mbuf_dealloc(_PyManagedBufferObject *self) } static int -mbuf_traverse(_PyManagedBufferObject *self, visitproc visit, void *arg) +mbuf_traverse(PyObject *_self, visitproc visit, void *arg) { + _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; Py_VISIT(self->master.obj); return 0; } static int -mbuf_clear(_PyManagedBufferObject *self) +mbuf_clear(PyObject *_self) { + _PyManagedBufferObject *self = (_PyManagedBufferObject *)_self; assert(self->exports >= 0); mbuf_release(self); return 0; @@ -148,7 +151,7 @@ PyTypeObject _PyManagedBuffer_Type = { "managedbuffer", sizeof(_PyManagedBufferObject), 0, - (destructor)mbuf_dealloc, /* tp_dealloc */ + mbuf_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -165,8 +168,8 @@ PyTypeObject _PyManagedBuffer_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)mbuf_traverse, /* tp_traverse */ - (inquiry)mbuf_clear /* tp_clear */ + mbuf_traverse, /* tp_traverse */ + mbuf_clear /* tp_clear */ }; @@ -1137,8 +1140,9 @@ memoryview_release_impl(PyMemoryViewObject *self) } static void -memory_dealloc(PyMemoryViewObject *self) +memory_dealloc(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; assert(self->exports == 0); _PyObject_GC_UNTRACK(self); (void)_memory_release(self); @@ -1149,15 +1153,17 @@ memory_dealloc(PyMemoryViewObject *self) } static int -memory_traverse(PyMemoryViewObject *self, visitproc visit, void *arg) +memory_traverse(PyObject *_self, visitproc visit, void *arg) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_VISIT(self->mbuf); return 0; } static int -memory_clear(PyMemoryViewObject *self) +memory_clear(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; (void)_memory_release(self); Py_CLEAR(self->mbuf); return 0; @@ -1510,8 +1516,9 @@ memoryview_toreadonly_impl(PyMemoryViewObject *self) /**************************************************************************/ static int -memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) +memory_getbuf(PyObject *_self, Py_buffer *view, int flags) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *base = &self->view; int baseflags = self->flags; @@ -1589,8 +1596,9 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) } static void -memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) +memory_releasebuf(PyObject *_self, Py_buffer *view) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; self->exports--; return; /* PyBuffer_Release() decrements view->obj after this function returns. */ @@ -1598,8 +1606,8 @@ memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) /* Buffer methods */ static PyBufferProcs memory_as_buffer = { - (getbufferproc)memory_getbuf, /* bf_getbuffer */ - (releasebufferproc)memory_releasebuf, /* bf_releasebuffer */ + memory_getbuf, /* bf_getbuffer */ + memory_releasebuf, /* bf_releasebuffer */ }; @@ -2344,8 +2352,9 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, } static PyObject * -memory_repr(PyMemoryViewObject *self) +memory_repr(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; if (self->flags & _Py_MEMORYVIEW_RELEASED) return PyUnicode_FromFormat("", self); else @@ -2421,8 +2430,9 @@ ptr_from_tuple(const Py_buffer *view, PyObject *tup) with the type specified by view->format. Otherwise, the item is a sub-view. The function is used in memory_subscript() and memory_as_sequence. */ static PyObject * -memory_item(PyMemoryViewObject *self, Py_ssize_t index) +memory_item(PyObject *_self, Py_ssize_t index) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &(self->view); const char *fmt; @@ -2546,8 +2556,9 @@ is_multiindex(PyObject *key) 0-d memoryview objects can be referenced using mv[...] or mv[()] but not with anything else. */ static PyObject * -memory_subscript(PyMemoryViewObject *self, PyObject *key) +memory_subscript(PyObject *_self, PyObject *key) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view; view = &(self->view); @@ -2575,7 +2586,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) index = PyNumber_AsSsize_t(key, PyExc_IndexError); if (index == -1 && PyErr_Occurred()) return NULL; - return memory_item(self, index); + return memory_item((PyObject *)self, index); } else if (PySlice_Check(key)) { CHECK_RESTRICTED(self); @@ -2608,8 +2619,9 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) } static int -memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) +memory_ass_sub(PyObject *_self, PyObject *key, PyObject *value) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &(self->view); Py_buffer src; const char *fmt; @@ -2710,8 +2722,9 @@ memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) } static Py_ssize_t -memory_length(PyMemoryViewObject *self) +memory_length(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED_INT(self); if (self->view.ndim == 0) { PyErr_SetString(PyExc_TypeError, "0-dim memory has no length"); @@ -2722,17 +2735,17 @@ memory_length(PyMemoryViewObject *self) /* As mapping */ static PyMappingMethods memory_as_mapping = { - (lenfunc)memory_length, /* mp_length */ - (binaryfunc)memory_subscript, /* mp_subscript */ - (objobjargproc)memory_ass_sub, /* mp_ass_subscript */ + memory_length, /* mp_length */ + memory_subscript, /* mp_subscript */ + memory_ass_sub, /* mp_ass_subscript */ }; /* As sequence */ static PySequenceMethods memory_as_sequence = { - (lenfunc)memory_length, /* sq_length */ + memory_length, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ - (ssizeargfunc)memory_item, /* sq_item */ + memory_item, /* sq_item */ }; @@ -3034,8 +3047,9 @@ memory_richcompare(PyObject *v, PyObject *w, int op) /**************************************************************************/ static Py_hash_t -memory_hash(PyMemoryViewObject *self) +memory_hash(PyObject *_self) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; if (self->hash == -1) { Py_buffer *view = &self->view; char *mem = view->buf; @@ -3112,8 +3126,9 @@ _IntTupleFromSsizet(int len, Py_ssize_t *vals) } static PyObject * -memory_obj_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_obj_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; Py_buffer *view = &self->view; CHECK_RELEASED(self); @@ -3124,78 +3139,89 @@ memory_obj_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) } static PyObject * -memory_nbytes_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_nbytes_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyLong_FromSsize_t(self->view.len); } static PyObject * -memory_format_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_format_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyUnicode_FromString(self->view.format); } static PyObject * -memory_itemsize_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_itemsize_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyLong_FromSsize_t(self->view.itemsize); } static PyObject * -memory_shape_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_shape_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return _IntTupleFromSsizet(self->view.ndim, self->view.shape); } static PyObject * -memory_strides_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_strides_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return _IntTupleFromSsizet(self->view.ndim, self->view.strides); } static PyObject * -memory_suboffsets_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_suboffsets_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return _IntTupleFromSsizet(self->view.ndim, self->view.suboffsets); } static PyObject * -memory_readonly_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_readonly_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(self->view.readonly); } static PyObject * -memory_ndim_get(PyMemoryViewObject *self, void *Py_UNUSED(ignored)) +memory_ndim_get(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyLong_FromLong(self->view.ndim); } static PyObject * -memory_c_contiguous(PyMemoryViewObject *self, PyObject *dummy) +memory_c_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(MV_C_CONTIGUOUS(self->flags)); } static PyObject * -memory_f_contiguous(PyMemoryViewObject *self, PyObject *dummy) +memory_f_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(MV_F_CONTIGUOUS(self->flags)); } static PyObject * -memory_contiguous(PyMemoryViewObject *self, PyObject *dummy) +memory_contiguous(PyObject *_self, void *Py_UNUSED(ignored)) { + PyMemoryViewObject *self = (PyMemoryViewObject *)_self; CHECK_RELEASED(self); return PyBool_FromLong(MV_ANY_CONTIGUOUS(self->flags)); } @@ -3232,18 +3258,18 @@ PyDoc_STRVAR(memory_contiguous_doc, static PyGetSetDef memory_getsetlist[] = { - {"obj", (getter)memory_obj_get, NULL, memory_obj_doc}, - {"nbytes", (getter)memory_nbytes_get, NULL, memory_nbytes_doc}, - {"readonly", (getter)memory_readonly_get, NULL, memory_readonly_doc}, - {"itemsize", (getter)memory_itemsize_get, NULL, memory_itemsize_doc}, - {"format", (getter)memory_format_get, NULL, memory_format_doc}, - {"ndim", (getter)memory_ndim_get, NULL, memory_ndim_doc}, - {"shape", (getter)memory_shape_get, NULL, memory_shape_doc}, - {"strides", (getter)memory_strides_get, NULL, memory_strides_doc}, - {"suboffsets", (getter)memory_suboffsets_get, NULL, memory_suboffsets_doc}, - {"c_contiguous", (getter)memory_c_contiguous, NULL, memory_c_contiguous_doc}, - {"f_contiguous", (getter)memory_f_contiguous, NULL, memory_f_contiguous_doc}, - {"contiguous", (getter)memory_contiguous, NULL, memory_contiguous_doc}, + {"obj", memory_obj_get, NULL, memory_obj_doc}, + {"nbytes", memory_nbytes_get, NULL, memory_nbytes_doc}, + {"readonly", memory_readonly_get, NULL, memory_readonly_doc}, + {"itemsize", memory_itemsize_get, NULL, memory_itemsize_doc}, + {"format", memory_format_get, NULL, memory_format_doc}, + {"ndim", memory_ndim_get, NULL, memory_ndim_doc}, + {"shape", memory_shape_get, NULL, memory_shape_doc}, + {"strides", memory_strides_get, NULL, memory_strides_doc}, + {"suboffsets", memory_suboffsets_get, NULL, memory_suboffsets_doc}, + {"c_contiguous", memory_c_contiguous, NULL, memory_c_contiguous_doc}, + {"f_contiguous", memory_f_contiguous, NULL, memory_f_contiguous_doc}, + {"contiguous", memory_contiguous, NULL, memory_contiguous_doc}, {NULL, NULL, NULL, NULL}, }; @@ -3276,23 +3302,26 @@ typedef struct { } memoryiterobject; static void -memoryiter_dealloc(memoryiterobject *it) +memoryiter_dealloc(PyObject *self) { + memoryiterobject *it = (memoryiterobject *)self; _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -memoryiter_traverse(memoryiterobject *it, visitproc visit, void *arg) +memoryiter_traverse(PyObject *self, visitproc visit, void *arg) { + memoryiterobject *it = (memoryiterobject *)self; Py_VISIT(it->it_seq); return 0; } static PyObject * -memoryiter_next(memoryiterobject *it) +memoryiter_next(PyObject *self) { + memoryiterobject *it = (memoryiterobject *)self; PyMemoryViewObject *seq; seq = it->it_seq; if (seq == NULL) { @@ -3347,7 +3376,7 @@ memory_iter(PyObject *seq) return NULL; } it->it_fmt = fmt; - it->it_length = memory_length(obj); + it->it_length = memory_length((PyObject *)obj); it->it_index = 0; it->it_seq = (PyMemoryViewObject*)Py_NewRef(obj); _PyObject_GC_TRACK(it); @@ -3359,12 +3388,12 @@ PyTypeObject _PyMemoryIter_Type = { .tp_name = "memory_iterator", .tp_basicsize = sizeof(memoryiterobject), // methods - .tp_dealloc = (destructor)memoryiter_dealloc, + .tp_dealloc = memoryiter_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)memoryiter_traverse, + .tp_traverse = memoryiter_traverse, .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)memoryiter_next, + .tp_iternext = memoryiter_next, }; PyTypeObject PyMemoryView_Type = { @@ -3372,16 +3401,16 @@ PyTypeObject PyMemoryView_Type = { "memoryview", /* tp_name */ offsetof(PyMemoryViewObject, ob_array), /* tp_basicsize */ sizeof(Py_ssize_t), /* tp_itemsize */ - (destructor)memory_dealloc, /* tp_dealloc */ + memory_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)memory_repr, /* tp_repr */ + memory_repr, /* tp_repr */ 0, /* tp_as_number */ &memory_as_sequence, /* tp_as_sequence */ &memory_as_mapping, /* tp_as_mapping */ - (hashfunc)memory_hash, /* tp_hash */ + memory_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -3390,8 +3419,8 @@ PyTypeObject PyMemoryView_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_SEQUENCE, /* tp_flags */ memoryview__doc__, /* tp_doc */ - (traverseproc)memory_traverse, /* tp_traverse */ - (inquiry)memory_clear, /* tp_clear */ + memory_traverse, /* tp_traverse */ + memory_clear, /* tp_clear */ memory_richcompare, /* tp_richcompare */ offsetof(PyMemoryViewObject, weakreflist),/* tp_weaklistoffset */ memory_iter, /* tp_iter */ diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index 4eb622ed4bad76..c50e3b05590b6f 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -123,6 +123,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) const bool force = collect >= MI_FORCE; _mi_deferred_free(heap, force); + // gh-112532: we may be called from a thread that is not the owner of the heap + bool is_main_thread = _mi_is_main_thread() && heap->thread_id == _mi_thread_id(); + // note: never reclaim on collect but leave it to threads that need storage to reclaim const bool force_main = #ifdef NDEBUG @@ -130,7 +133,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) #else collect >= MI_FORCE #endif - && _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim; + && is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim; if (force_main) { // the main thread is abandoned (end-of-program), try to reclaim all abandoned segments. @@ -164,7 +167,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) } // collect regions on program-exit (or shared library unload) - if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) { + if (force && is_main_thread && mi_heap_is_backing(heap)) { _mi_thread_data_collect(); // collect thread data cache _mi_arena_collect(true /* force purge */, &heap->tld->stats); } @@ -206,18 +209,28 @@ mi_heap_t* mi_heap_get_backing(void) { return bheap; } -mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { - mi_heap_t* bheap = mi_heap_get_backing(); - mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? - if (heap == NULL) return NULL; +void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id) +{ _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); - heap->tld = bheap->tld; + heap->tld = tld; heap->thread_id = _mi_thread_id(); heap->arena_id = arena_id; - _mi_random_split(&bheap->random, &heap->random); + if (heap == tld->heap_backing) { + _mi_random_init(&heap->random); + } + else { + _mi_random_split(&tld->heap_backing->random, &heap->random); + } heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); +} + +mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { + mi_heap_t* bheap = mi_heap_get_backing(); + mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? + if (heap == NULL) return NULL; + _mi_heap_init_ex(heap, bheap->tld, arena_id); heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe // push on the thread local heaps list heap->next = heap->tld->heaps; diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c index 7dfa7657737117..376e14b49b7c7b 100644 --- a/Objects/mimalloc/init.c +++ b/Objects/mimalloc/init.c @@ -297,24 +297,20 @@ static bool _mi_heap_init(void) { mi_thread_data_t* td = mi_thread_data_zalloc(); if (td == NULL) return false; - mi_tld_t* tld = &td->tld; - mi_heap_t* heap = &td->heap; + _mi_tld_init(&td->tld, &td->heap); + _mi_heap_init_ex(&td->heap, &td->tld, _mi_arena_id_none()); + _mi_heap_set_default_direct(&td->heap); + } + return false; +} + +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { _mi_memcpy_aligned(tld, &tld_empty, sizeof(*tld)); - _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap)); - heap->thread_id = _mi_thread_id(); - _mi_random_init(&heap->random); - heap->cookie = _mi_heap_random_next(heap) | 1; - heap->keys[0] = _mi_heap_random_next(heap); - heap->keys[1] = _mi_heap_random_next(heap); - heap->tld = tld; - tld->heap_backing = heap; - tld->heaps = heap; tld->segments.stats = &tld->stats; tld->segments.os = &tld->os; tld->os.stats = &tld->stats; - _mi_heap_set_default_direct(heap); - } - return false; + tld->heap_backing = bheap; + tld->heaps = bheap; } // Free the thread local default heap (called from `mi_thread_done`) diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c index cffbb2d0b4d7b2..2152017e01fb43 100644 --- a/Objects/mimalloc/prim/unix/prim.c +++ b/Objects/mimalloc/prim/unix/prim.c @@ -170,7 +170,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { int err = errno; - _mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); + _mi_verbose_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); } if (p!=MAP_FAILED) return p; // fall back to regular mmap @@ -195,7 +195,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p #else int err = errno; #endif - _mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + _mi_verbose_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); } if (p!=MAP_FAILED) return p; // fall back to regular mmap diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index bba77ce8ab7e7b..3a1c516658dce7 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -749,27 +749,20 @@ module_repr(PyModuleObject *m) } /* Check if the "_initializing" attribute of the module spec is set to true. - Clear the exception and return 0 if spec is NULL. */ int _PyModuleSpec_IsInitializing(PyObject *spec) { - if (spec != NULL) { - PyObject *value; - int ok = PyObject_GetOptionalAttr(spec, &_Py_ID(_initializing), &value); - if (ok == 0) { - return 0; - } - if (value != NULL) { - int initializing = PyObject_IsTrue(value); - Py_DECREF(value); - if (initializing >= 0) { - return initializing; - } - } + if (spec == NULL) { + return 0; } - PyErr_Clear(); - return 0; + PyObject *value; + int rc = PyObject_GetOptionalAttr(spec, &_Py_ID(_initializing), &value); + if (rc > 0) { + rc = PyObject_IsTrue(value); + Py_DECREF(value); + } + return rc; } /* Check if the submodule name is in the "_uninitialized_submodules" attribute @@ -782,24 +775,20 @@ _PyModuleSpec_IsUninitializedSubmodule(PyObject *spec, PyObject *name) return 0; } - PyObject *value = PyObject_GetAttr(spec, &_Py_ID(_uninitialized_submodules)); - if (value == NULL) { - return 0; + PyObject *value; + int rc = PyObject_GetOptionalAttr(spec, &_Py_ID(_uninitialized_submodules), &value); + if (rc > 0) { + rc = PySequence_Contains(value, name); + Py_DECREF(value); } - - int is_uninitialized = PySequence_Contains(value, name); - Py_DECREF(value); - if (is_uninitialized == -1) { - return 0; - } - return is_uninitialized; + return rc; } PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { // When suppress=1, this function suppresses AttributeError. - PyObject *attr, *mod_name, *getattr; + PyObject *attr, *mod_name, *getattr, *origin; attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr) { return attr; @@ -840,23 +829,47 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) return NULL; } if (suppress != 1) { - if (_PyModuleSpec_IsInitializing(spec)) { - PyErr_Format(PyExc_AttributeError, + int rc = _PyModuleSpec_IsInitializing(spec); + if (rc > 0) { + int valid_spec = PyObject_GetOptionalAttr(spec, &_Py_ID(origin), &origin); + if (valid_spec == -1) { + Py_XDECREF(spec); + Py_DECREF(mod_name); + return NULL; + } + if (valid_spec == 1 && !PyUnicode_Check(origin)) { + valid_spec = 0; + Py_DECREF(origin); + } + if (valid_spec == 1) { + PyErr_Format(PyExc_AttributeError, + "partially initialized " + "module '%U' from '%U' has no attribute '%U' " + "(most likely due to a circular import)", + mod_name, origin, name); + Py_DECREF(origin); + } + else { + PyErr_Format(PyExc_AttributeError, "partially initialized " "module '%U' has no attribute '%U' " "(most likely due to a circular import)", mod_name, name); + } } - else if (_PyModuleSpec_IsUninitializedSubmodule(spec, name)) { - PyErr_Format(PyExc_AttributeError, + else if (rc == 0) { + rc = _PyModuleSpec_IsUninitializedSubmodule(spec, name); + if (rc > 0) { + PyErr_Format(PyExc_AttributeError, "cannot access submodule '%U' of module '%U' " "(most likely due to a circular import)", name, mod_name); - } - else { - PyErr_Format(PyExc_AttributeError, + } + else if (rc == 0) { + PyErr_Format(PyExc_AttributeError, "module '%U' has no attribute '%U'", mod_name, name); + } } } Py_XDECREF(spec); diff --git a/Objects/object.c b/Objects/object.c index d145674cb3ba34..d970a26756173b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1196,7 +1196,7 @@ PyObject_GetOptionalAttr(PyObject *v, PyObject *name, PyObject **result) } return 0; } - if (tp->tp_getattro == (getattrofunc)_Py_type_getattro) { + if (tp->tp_getattro == _Py_type_getattro) { int supress_missing_attribute_exception = 0; *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &supress_missing_attribute_exception); if (supress_missing_attribute_exception) { @@ -2026,7 +2026,7 @@ PyTypeObject _PyNone_Type = { 0, /*tp_doc */ 0, /*tp_traverse */ 0, /*tp_clear */ - 0, /*tp_richcompare */ + _Py_BaseObject_RichCompare, /*tp_richcompare */ 0, /*tp_weaklistoffset */ 0, /*tp_iter */ 0, /*tp_iternext */ diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 2761c774209786..883adcb1c19b6e 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -16,6 +16,10 @@ # include "mimalloc/internal.h" // for stats #endif +#if defined(Py_GIL_DISABLED) && !defined(WITH_MIMALLOC) +# error "Py_GIL_DISABLED requires WITH_MIMALLOC" +#endif + #undef uint #define uint pymem_uint @@ -84,19 +88,37 @@ _PyMem_RawFree(void *Py_UNUSED(ctx), void *ptr) void * _PyMem_MiMalloc(void *ctx, size_t size) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_malloc(heap, size); +#else return mi_malloc(size); +#endif } void * _PyMem_MiCalloc(void *ctx, size_t nelem, size_t elsize) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_calloc(heap, nelem, elsize); +#else return mi_calloc(nelem, elsize); +#endif } void * _PyMem_MiRealloc(void *ctx, void *ptr, size_t size) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_realloc(heap, ptr, size); +#else return mi_realloc(ptr, size); +#endif } void @@ -108,20 +130,38 @@ _PyMem_MiFree(void *ctx, void *ptr) void * _PyObject_MiMalloc(void *ctx, size_t nbytes) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_malloc(heap, nbytes); +#else return mi_malloc(nbytes); +#endif } void * _PyObject_MiCalloc(void *ctx, size_t nelem, size_t elsize) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_calloc(heap, nelem, elsize); +#else return mi_calloc(nelem, elsize); +#endif } void * _PyObject_MiRealloc(void *ctx, void *ptr, size_t nbytes) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_realloc(heap, ptr, nbytes); +#else return mi_realloc(ptr, nbytes); +#endif } void @@ -153,7 +193,12 @@ void* _PyObject_Realloc(void *ctx, void *ptr, size_t size); # define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free} #endif // WITH_PYMALLOC -#if defined(WITH_PYMALLOC) +#if defined(Py_GIL_DISABLED) +// Py_GIL_DISABLED requires using mimalloc for "mem" and "obj" domains. +# define PYRAW_ALLOC MALLOC_ALLOC +# define PYMEM_ALLOC MIMALLOC_ALLOC +# define PYOBJ_ALLOC MIMALLOC_OBJALLOC +#elif defined(WITH_PYMALLOC) # define PYRAW_ALLOC MALLOC_ALLOC # define PYMEM_ALLOC PYMALLOC_ALLOC # define PYOBJ_ALLOC PYMALLOC_ALLOC @@ -329,13 +374,9 @@ int _PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *old_alloc) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must be initializing. */ - return set_default_allocator_unlocked(domain, pydebug, old_alloc); - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); int res = set_default_allocator_unlocked(domain, pydebug, old_alloc); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); return res; } @@ -354,7 +395,7 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator) else if (strcmp(name, "debug") == 0) { *allocator = PYMEM_ALLOCATOR_DEBUG; } -#ifdef WITH_PYMALLOC +#if defined(WITH_PYMALLOC) && !defined(Py_GIL_DISABLED) else if (strcmp(name, "pymalloc") == 0) { *allocator = PYMEM_ALLOCATOR_PYMALLOC; } @@ -370,12 +411,14 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator) *allocator = PYMEM_ALLOCATOR_MIMALLOC_DEBUG; } #endif +#ifndef Py_GIL_DISABLED else if (strcmp(name, "malloc") == 0) { *allocator = PYMEM_ALLOCATOR_MALLOC; } else if (strcmp(name, "malloc_debug") == 0) { *allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG; } +#endif else { /* unknown allocator */ return -1; @@ -467,9 +510,9 @@ set_up_allocators_unlocked(PyMemAllocatorName allocator) int _PyMem_SetupAllocators(PyMemAllocatorName allocator) { - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); int res = set_up_allocators_unlocked(allocator); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); return res; } @@ -554,9 +597,9 @@ get_current_allocator_name_unlocked(void) const char* _PyMem_GetCurrentAllocatorName(void) { - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); const char *name = get_current_allocator_name_unlocked(); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); return name; } @@ -653,14 +696,9 @@ set_up_debug_hooks_unlocked(void) void PyMem_SetupDebugHooks(void) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - set_up_debug_hooks_unlocked(); - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); set_up_debug_hooks_unlocked(); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } static void @@ -696,53 +734,33 @@ set_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - get_allocator_unlocked(domain, allocator); - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); get_allocator_unlocked(domain, allocator); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - set_allocator_unlocked(domain, allocator); - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); set_allocator_unlocked(domain, allocator); - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - *allocator = _PyObject_Arena; - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); *allocator = _PyObject_Arena; - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) { - if (ALLOCATORS_MUTEX == NULL) { - /* The runtime must not be completely initialized yet. */ - _PyObject_Arena = *allocator; - return; - } - PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK); + PyMutex_Lock(&ALLOCATORS_MUTEX); _PyObject_Arena = *allocator; - PyThread_release_lock(ALLOCATORS_MUTEX); + PyMutex_Unlock(&ALLOCATORS_MUTEX); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index aa00e04ad5e11b..ea29a38d74ae3e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1597,8 +1597,9 @@ static PyGetSetDef type_getsets[] = { }; static PyObject * -type_repr(PyTypeObject *type) +type_repr(PyObject *self) { + PyTypeObject *type = (PyTypeObject *)self; if (type->tp_name == NULL) { // type_repr() called before the type is fully initialized // by PyType_Ready(). @@ -1630,8 +1631,9 @@ type_repr(PyTypeObject *type) } static PyObject * -type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) +type_call(PyObject *self, PyObject *args, PyObject *kwds) { + PyTypeObject *type = (PyTypeObject *)self; PyObject *obj; PyThreadState *tstate = _PyThreadState_GET(); @@ -4917,14 +4919,15 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin /* This is similar to PyObject_GenericGetAttr(), but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ PyObject * -_Py_type_getattro(PyTypeObject *type, PyObject *name) +_Py_type_getattro(PyObject *type, PyObject *name) { - return _Py_type_getattro_impl(type, name, NULL); + return _Py_type_getattro_impl((PyTypeObject *)type, name, NULL); } static int -type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) +type_setattro(PyObject *self, PyObject *name, PyObject *value) { + PyTypeObject *type = (PyTypeObject *)self; int res; if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { PyErr_Format( @@ -5069,8 +5072,10 @@ _PyStaticType_Dealloc(PyInterpreterState *interp, PyTypeObject *type) static void -type_dealloc(PyTypeObject *type) +type_dealloc(PyObject *self) { + PyTypeObject *type = (PyTypeObject *)self; + // Assert this is a heap-allocated type object _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -5257,8 +5262,10 @@ PyDoc_STRVAR(type_doc, "type(name, bases, dict, **kwds) -> a new type"); static int -type_traverse(PyTypeObject *type, visitproc visit, void *arg) +type_traverse(PyObject *self, visitproc visit, void *arg) { + PyTypeObject *type = (PyTypeObject *)self; + /* Because of type_is_gc(), the collector only calls this for heaptypes. */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { @@ -5286,8 +5293,10 @@ type_traverse(PyTypeObject *type, visitproc visit, void *arg) } static int -type_clear(PyTypeObject *type) +type_clear(PyObject *self) { + PyTypeObject *type = (PyTypeObject *)self; + /* Because of type_is_gc(), the collector only calls this for heaptypes. */ _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -5334,9 +5343,9 @@ type_clear(PyTypeObject *type) } static int -type_is_gc(PyTypeObject *type) +type_is_gc(PyObject *type) { - return type->tp_flags & Py_TPFLAGS_HEAPTYPE; + return ((PyTypeObject *)type)->tp_flags & Py_TPFLAGS_HEAPTYPE; } @@ -5349,28 +5358,28 @@ PyTypeObject PyType_Type = { "type", /* tp_name */ sizeof(PyHeapTypeObject), /* tp_basicsize */ sizeof(PyMemberDef), /* tp_itemsize */ - (destructor)type_dealloc, /* tp_dealloc */ + type_dealloc, /* tp_dealloc */ offsetof(PyTypeObject, tp_vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)type_repr, /* tp_repr */ + type_repr, /* tp_repr */ &type_as_number, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)type_call, /* tp_call */ + type_call, /* tp_call */ 0, /* tp_str */ - (getattrofunc)_Py_type_getattro, /* tp_getattro */ - (setattrofunc)type_setattro, /* tp_setattro */ + _Py_type_getattro, /* tp_getattro */ + type_setattro, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS | Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_ITEMS_AT_END, /* tp_flags */ type_doc, /* tp_doc */ - (traverseproc)type_traverse, /* tp_traverse */ - (inquiry)type_clear, /* tp_clear */ + type_traverse, /* tp_traverse */ + type_clear, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyTypeObject, tp_weaklist), /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -5387,7 +5396,7 @@ PyTypeObject PyType_Type = { 0, /* tp_alloc */ type_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ - (inquiry)type_is_gc, /* tp_is_gc */ + type_is_gc, /* tp_is_gc */ .tp_vectorcall = type_vectorcall, }; @@ -5625,6 +5634,12 @@ object_richcompare(PyObject *self, PyObject *other, int op) return res; } +PyObject* +_Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op) +{ + return object_richcompare(self, other, op); +} + static PyObject * object_get_class(PyObject *self, void *closure) { @@ -6555,6 +6570,12 @@ PyDoc_STRVAR(object_doc, "When called, it accepts no arguments and returns a new featureless\n" "instance that has no instance attributes and cannot be given any.\n"); +static Py_hash_t +object_hash(PyObject *obj) +{ + return _Py_HashPointer(obj); +} + PyTypeObject PyBaseObject_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "object", /* tp_name */ @@ -6569,7 +6590,7 @@ PyTypeObject PyBaseObject_Type = { 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)_Py_HashPointer, /* tp_hash */ + object_hash, /* tp_hash */ 0, /* tp_call */ object_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -10383,9 +10404,22 @@ supercheck(PyTypeObject *type, PyObject *obj) Py_XDECREF(class_attr); } - PyErr_SetString(PyExc_TypeError, - "super(type, obj): " - "obj must be an instance or subtype of type"); + const char *type_or_instance, *obj_str; + + if (PyType_Check(obj)) { + type_or_instance = "type"; + obj_str = ((PyTypeObject*)obj)->tp_name; + } + else { + type_or_instance = "instance of"; + obj_str = Py_TYPE(obj)->tp_name; + } + + PyErr_Format(PyExc_TypeError, + "super(type, obj): obj (%s %.200s) is not " + "an instance or subtype of type (%.200s).", + type_or_instance, obj_str, type->tp_name); + return NULL; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index cffc06297a9aee..4b03cc3f4da5fa 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1897,6 +1897,7 @@ PyUnicode_FromString(const char *u) PyObject * _PyUnicode_FromId(_Py_Identifier *id) { + PyMutex_Lock((PyMutex *)&id->mutex); PyInterpreterState *interp = _PyInterpreterState_GET(); struct _Py_unicode_ids *ids = &interp->unicode.ids; @@ -1904,7 +1905,7 @@ _PyUnicode_FromId(_Py_Identifier *id) if (index < 0) { struct _Py_unicode_runtime_ids *rt_ids = &interp->runtime->unicode_state.ids; - PyThread_acquire_lock(rt_ids->lock, WAIT_LOCK); + PyMutex_Lock(&rt_ids->mutex); // Check again to detect concurrent access. Another thread can have // initialized the index while this thread waited for the lock. index = _Py_atomic_load_ssize(&id->index); @@ -1914,7 +1915,7 @@ _PyUnicode_FromId(_Py_Identifier *id) rt_ids->next_index++; _Py_atomic_store_ssize(&id->index, index); } - PyThread_release_lock(rt_ids->lock); + PyMutex_Unlock(&rt_ids->mutex); } assert(index >= 0); @@ -1923,14 +1924,14 @@ _PyUnicode_FromId(_Py_Identifier *id) obj = ids->array[index]; if (obj) { // Return a borrowed reference - return obj; + goto end; } } obj = PyUnicode_DecodeUTF8Stateful(id->string, strlen(id->string), NULL, NULL); if (!obj) { - return NULL; + goto end; } PyUnicode_InternInPlace(&obj); @@ -1941,7 +1942,8 @@ _PyUnicode_FromId(_Py_Identifier *id) PyObject **new_array = PyMem_Realloc(ids->array, new_size * item_size); if (new_array == NULL) { PyErr_NoMemory(); - return NULL; + obj = NULL; + goto end; } memset(&new_array[ids->size], 0, (new_size - ids->size) * item_size); ids->array = new_array; @@ -1951,6 +1953,8 @@ _PyUnicode_FromId(_Py_Identifier *id) // The array stores a strong reference ids->array[index] = obj; +end: + PyMutex_Unlock((PyMutex *)&id->mutex); // Return a borrowed reference return obj; } @@ -5869,6 +5873,23 @@ PyUnicode_AsUTF16String(PyObject *unicode) return _PyUnicode_EncodeUTF16(unicode, NULL, 0); } +_PyUnicode_Name_CAPI * +_PyUnicode_GetNameCAPI(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_Name_CAPI *ucnhash_capi; + + ucnhash_capi = _Py_atomic_load_ptr(&interp->unicode.ucnhash_capi); + if (ucnhash_capi == NULL) { + ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( + PyUnicodeData_CAPSULE_NAME, 1); + + // It's fine if we overwite the value here. It's always the same value. + _Py_atomic_store_ptr(&interp->unicode.ucnhash_capi, ucnhash_capi); + } + return ucnhash_capi; +} + /* --- Unicode Escape Codec ----------------------------------------------- */ PyObject * @@ -5884,7 +5905,6 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, PyObject *errorHandler = NULL; PyObject *exc = NULL; _PyUnicode_Name_CAPI *ucnhash_capi; - PyInterpreterState *interp = _PyInterpreterState_GET(); // so we can remember if we've seen an invalid escape char or not *first_invalid_escape = NULL; @@ -6032,19 +6052,13 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, /* \N{name} */ case 'N': - ucnhash_capi = interp->unicode.ucnhash_capi; + ucnhash_capi = _PyUnicode_GetNameCAPI(); if (ucnhash_capi == NULL) { - /* load the unicode data module */ - ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( - PyUnicodeData_CAPSULE_NAME, 1); - if (ucnhash_capi == NULL) { - PyErr_SetString( + PyErr_SetString( PyExc_UnicodeError, "\\N escapes not supported (can't load unicodedata module)" - ); - goto onError; - } - interp->unicode.ucnhash_capi = ucnhash_capi; + ); + goto onError; } message = "malformed \\N character escape"; @@ -12494,11 +12508,13 @@ str.split as unicode_split character (including \n \r \t \f and spaces) and will discard empty strings from the result. maxsplit: Py_ssize_t = -1 - Maximum number of splits (starting from the left). + Maximum number of splits. -1 (the default value) means no limit. Return a list of the substrings in the string, using sep as the separator string. +Splitting starts at the front of the string and works to the end. + Note, str.split() is mainly useful for data that has been intentionally delimited. With natural text that includes punctuation, consider using the regular expression module. @@ -12507,7 +12523,7 @@ the regular expression module. static PyObject * unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=3a65b1db356948dc input=07b9040d98c5fe8d]*/ +/*[clinic end generated code: output=3a65b1db356948dc input=a29bcc0c7a5af0eb]*/ { if (sep == Py_None) return split(self, NULL, maxsplit); diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index fdf09ec6ec6f63..5ab6dcb032550b 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -44,6 +44,8 @@ struct _query_data { LPCWSTR query; HANDLE writePipe; HANDLE readPipe; + HANDLE initEvent; + HANDLE connectEvent; }; @@ -80,12 +82,18 @@ _query_thread(LPVOID param) IID_IWbemLocator, (LPVOID *)&locator ); } + if (SUCCEEDED(hr) && !SetEvent(data->initEvent)) { + hr = HRESULT_FROM_WIN32(GetLastError()); + } if (SUCCEEDED(hr)) { hr = locator->ConnectServer( bstr_t(L"ROOT\\CIMV2"), NULL, NULL, 0, NULL, 0, 0, &services ); } + if (SUCCEEDED(hr) && !SetEvent(data->connectEvent)) { + hr = HRESULT_FROM_WIN32(GetLastError()); + } if (SUCCEEDED(hr)) { hr = CoSetProxyBlanket( services, RPC_C_AUTHN_WINNT, RPC_C_AUTHZ_NONE, NULL, @@ -189,6 +197,24 @@ _query_thread(LPVOID param) } +static DWORD +wait_event(HANDLE event, DWORD timeout) +{ + DWORD err = 0; + switch (WaitForSingleObject(event, timeout)) { + case WAIT_OBJECT_0: + break; + case WAIT_TIMEOUT: + err = WAIT_TIMEOUT; + break; + default: + err = GetLastError(); + break; + } + return err; +} + + /*[clinic input] _wmi.exec_query @@ -231,7 +257,11 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) Py_BEGIN_ALLOW_THREADS - if (!CreatePipe(&data.readPipe, &data.writePipe, NULL, 0)) { + data.initEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + data.connectEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if (!data.initEvent || !data.connectEvent || + !CreatePipe(&data.readPipe, &data.writePipe, NULL, 0)) + { err = GetLastError(); } else { hThread = CreateThread(NULL, 0, _query_thread, (LPVOID*)&data, 0, NULL); @@ -243,6 +273,17 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) } } + // gh-112278: If current user doesn't have permission to query the WMI, the + // function IWbemLocator::ConnectServer will hang for 5 seconds, and there + // is no way to specify the timeout. So we use an Event object to simulate + // a timeout. The initEvent will be set after COM initialization, it will + // take a longer time when first initialized. The connectEvent will be set + // after connected to WMI. + err = wait_event(data.initEvent, 1000); + if (!err) { + err = wait_event(data.connectEvent, 100); + } + while (!err) { if (ReadFile( data.readPipe, @@ -265,7 +306,7 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) } // Allow the thread some time to clean up - switch (WaitForSingleObject(hThread, 1000)) { + switch (WaitForSingleObject(hThread, 100)) { case WAIT_OBJECT_0: // Thread ended cleanly if (!GetExitCodeThread(hThread, (LPDWORD)&err)) { @@ -286,6 +327,8 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) } CloseHandle(hThread); + CloseHandle(data.initEvent); + CloseHandle(data.connectEvent); hThread = NULL; Py_END_ALLOW_THREADS diff --git a/PC/config.c b/PC/config.c index da2bde640961e0..f754ce6d3b057b 100644 --- a/PC/config.c +++ b/PC/config.c @@ -37,6 +37,7 @@ extern PyObject* PyInit__weakref(void); extern PyObject* PyInit_xxsubtype(void); extern PyObject* PyInit__xxsubinterpreters(void); extern PyObject* PyInit__xxinterpchannels(void); +extern PyObject* PyInit__xxinterpqueues(void); extern PyObject* PyInit__random(void); extern PyObject* PyInit_itertools(void); extern PyObject* PyInit__collections(void); @@ -142,6 +143,7 @@ struct _inittab _PyImport_Inittab[] = { {"xxsubtype", PyInit_xxsubtype}, {"_xxsubinterpreters", PyInit__xxsubinterpreters}, {"_xxinterpchannels", PyInit__xxinterpchannels}, + {"_xxinterpqueues", PyInit__xxinterpqueues}, #ifdef _Py_HAVE_ZLIB {"zlib", PyInit_zlib}, #endif diff --git a/PC/launcher2.c b/PC/launcher2.c index 116091f01227b8..2a8f8a101fc8a6 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -438,7 +438,7 @@ typedef struct { bool list; // if true, only list detected runtimes with paths without launching bool listPaths; - // if true, display help message before contiuning + // if true, display help message before continuing bool help; // if set, limits search to registry keys with the specified Company // This is intended for debugging and testing only diff --git a/PC/layout/main.py b/PC/layout/main.py index cb2e4878da26b1..accfd51dd978fb 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -73,7 +73,10 @@ def copy_if_modified(src, dest): ) if do_copy: - shutil.copy2(src, dest) + try: + shutil.copy2(src, dest) + except FileNotFoundError: + raise FileNotFoundError(src) from None def get_lib_layout(ns): @@ -208,8 +211,7 @@ def _c(d): for dest, src in rglob(ns.source / "Include", "**/*.h"): yield "include/{}".format(dest), src - src = ns.source / "PC" / "pyconfig.h" - yield "include/pyconfig.h", src + yield "include/pyconfig.h", ns.build / "pyconfig.h" for dest, src in get_tcltk_lib(ns): yield dest, src diff --git a/PC/pyconfig.h b/PC/pyconfig.h.in similarity index 99% rename from PC/pyconfig.h rename to PC/pyconfig.h.in index e6b368caffe280..d8f0a6be69c21a 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h.in @@ -739,4 +739,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */ #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1 +/* Define if you want to disable the GIL */ +#undef Py_GIL_DISABLED + #endif /* !Py_CONFIG_H */ diff --git a/PC/winsound.c b/PC/winsound.c index b0e416cfec4699..7e4ebd90f50c2e 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -35,6 +35,8 @@ winsound.PlaySound(None, 0) */ +#include "pyconfig.h" // Py_GIL_DISABLED + #ifndef Py_GIL_DISABLED // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #define Py_LIMITED_API 0x030c0000 diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a1c37e183f21c7..292bfa76519507 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -89,6 +89,7 @@ Py_NO_ENABLE_SHARED;Py_BUILD_CORE;_CONSOLE;%(PreprocessorDefinitions) + $(IntDir);%(AdditionalIncludeDirectories) Disabled false @@ -257,122 +258,125 @@ + + + importlib._bootstrap $(IntDir)importlib._bootstrap.g.h - $(PySourcePath)Python\frozen_modules\importlib._bootstrap.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap.h importlib._bootstrap_external $(IntDir)importlib._bootstrap_external.g.h - $(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap_external.h zipimport $(IntDir)zipimport.g.h - $(PySourcePath)Python\frozen_modules\zipimport.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\zipimport.h abc $(IntDir)abc.g.h - $(PySourcePath)Python\frozen_modules\abc.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\abc.h codecs $(IntDir)codecs.g.h - $(PySourcePath)Python\frozen_modules\codecs.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h io $(IntDir)io.g.h - $(PySourcePath)Python\frozen_modules\io.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\io.h _collections_abc $(IntDir)_collections_abc.g.h - $(PySourcePath)Python\frozen_modules\_collections_abc.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\_collections_abc.h _sitebuiltins $(IntDir)_sitebuiltins.g.h - $(PySourcePath)Python\frozen_modules\_sitebuiltins.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\_sitebuiltins.h genericpath $(IntDir)genericpath.g.h - $(PySourcePath)Python\frozen_modules\genericpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\genericpath.h ntpath $(IntDir)ntpath.g.h - $(PySourcePath)Python\frozen_modules\ntpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\ntpath.h posixpath $(IntDir)posixpath.g.h - $(PySourcePath)Python\frozen_modules\posixpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\posixpath.h os $(IntDir)os.g.h - $(PySourcePath)Python\frozen_modules\os.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\os.h site $(IntDir)site.g.h - $(PySourcePath)Python\frozen_modules\site.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\site.h stat $(IntDir)stat.g.h - $(PySourcePath)Python\frozen_modules\stat.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\stat.h importlib.util $(IntDir)importlib.util.g.h - $(PySourcePath)Python\frozen_modules\importlib.util.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.util.h importlib.machinery $(IntDir)importlib.machinery.g.h - $(PySourcePath)Python\frozen_modules\importlib.machinery.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.machinery.h runpy $(IntDir)runpy.g.h - $(PySourcePath)Python\frozen_modules\runpy.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\runpy.h __hello__ $(IntDir)__hello__.g.h - $(PySourcePath)Python\frozen_modules\__hello__.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__hello__.h __phello__ $(IntDir)__phello__.g.h - $(PySourcePath)Python\frozen_modules\__phello__.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.h __phello__.ham $(IntDir)__phello__.ham.g.h - $(PySourcePath)Python\frozen_modules\__phello__.ham.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.h __phello__.ham.eggs $(IntDir)__phello__.ham.eggs.g.h - $(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.eggs.h __phello__.spam $(IntDir)__phello__.spam.g.h - $(PySourcePath)Python\frozen_modules\__phello__.spam.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.spam.h frozen_only $(IntDir)frozen_only.g.h - $(PySourcePath)Python\frozen_modules\frozen_only.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\frozen_only.h @@ -381,39 +385,65 @@ getpath $(IntDir)getpath.g.h - $(PySourcePath)Python\frozen_modules\getpath.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\getpath.h - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @(PyConfigH->'%(FullPath)', ';') + $([System.IO.File]::ReadAllText($(PyConfigH))) + $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) + + + $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + + + + + @@ -454,7 +484,7 @@ $(IntDir)\deepfreeze_mappings.txt Overwrite="true" Lines="@(FrozenModule->'%(FullPath):%(FrozenId)')" /> - + @@ -463,7 +493,7 @@ $(IntDir)\deepfreeze_mappings.txt - + diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 94437f054d788c..3919c0592ec00d 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,9 +54,9 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.11 -set libraries=%libraries% sqlite-3.43.1.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 +set libraries=%libraries% sqlite-3.44.2.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 set libraries=%libraries% xz-5.2.5 set libraries=%libraries% zlib-1.2.13 @@ -77,7 +77,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.11 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.0 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.1 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 0acc7045c39a26..d69b43b0406ce0 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -10,6 +10,9 @@ $(MSBuildThisFileDirectory)obj\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\ $(IntDir.Replace(`\\`, `\`)) + + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen_$(Configuration)\ $(ProjectName) $(TargetName)$(PyDebugExt) false @@ -38,9 +41,8 @@ - $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories) + $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories) WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) - Py_GIL_DISABLED=1;%(PreprocessorDefinitions) _Py_USING_PGO=1;%(PreprocessorDefinitions) MaxSpeed @@ -60,6 +62,7 @@ -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) -flto %(AdditionalOptions) -d2pattern-opt-disable:-932189325 %(AdditionalOptions) + /sourceDependencies "$(IntDir.Trim(`\`))" %(AdditionalOptions) OnlyExplicitInline diff --git a/PCbuild/python.props b/PCbuild/python.props index 496bc3dd4cf794..3b7a8876a707df 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -68,7 +68,7 @@ - $(ExternalsDir)sqlite-3.43.1.0\ + $(ExternalsDir)sqlite-3.44.2.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ $(ExternalsDir)libffi-3.4.4\ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index be1b98dba02fc5..be5b34220aa0bc 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -111,6 +111,7 @@ + $(GeneratedFrozenModulesDir);%(AdditionalIncludeDirectories) PREFIX=NULL; EXEC_PREFIX=NULL; @@ -285,6 +286,7 @@ + @@ -377,7 +379,7 @@ - + @@ -457,6 +459,7 @@ + @@ -559,7 +562,9 @@ - + + $(GeneratedFrozenModulesDir)Python;%(AdditionalIncludeDirectories) + @@ -614,7 +619,7 @@ - + @@ -644,6 +649,35 @@ + + + + + + + + + + @(PyConfigH->'%(FullPath)', ';') + $([System.IO.File]::ReadAllText($(PyConfigH))) + $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) + + + $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + + + + + + + + + + + git diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4f0da8f35998b7..a96ca24cf08b66 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -780,6 +780,9 @@ Include\internal + + Include\internal + Include\internal @@ -1502,6 +1505,9 @@ Modules + + Modules + Parser diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 175ce918ac20f6..b9d76515c383f7 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -189,7 +189,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.43.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.44.2, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 96dd289face6a5..8ddf01d5dd1dca 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 8.6.13.0 + 8.6.13.1 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index c9bf08ed2e0f6d..4bb337349748cf 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -731,7 +731,7 @@ def emit_sequence_constructor(self, name, type): class PyTypesDeclareVisitor(PickleVisitor): def visitProduct(self, prod, name): - self.emit("static PyObject* ast2obj_%s(struct ast_state *state, void*);" % name, 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, void*);" % name, 0) if prod.attributes: self.emit("static const char * const %s_attributes[] = {" % name, 0) for a in prod.attributes: @@ -752,7 +752,7 @@ def visitSum(self, sum, name): ptype = "void*" if is_simple(sum): ptype = get_c_type(name) - self.emit("static PyObject* ast2obj_%s(struct ast_state *state, %s);" % (name, ptype), 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, %s);" % (name, ptype), 0) for t in sum.types: self.visitConstructor(t, name) @@ -984,7 +984,8 @@ def visitModule(self, mod): /* Conversion AST -> Python */ -static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, struct validator *vstate, asdl_seq *seq, + PyObject* (*func)(struct ast_state *state, struct validator *vstate, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -992,7 +993,7 @@ def visitModule(self, mod): if (!result) return NULL; for (i = 0; i < n; i++) { - value = func(state, asdl_seq_GET_UNTYPED(seq, i)); + value = func(state, vstate, asdl_seq_GET_UNTYPED(seq, i)); if (!value) { Py_DECREF(result); return NULL; @@ -1002,7 +1003,7 @@ def visitModule(self, mod): return result; } -static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), void *o) { PyObject *op = (PyObject*)o; if (!op) { @@ -1014,7 +1015,7 @@ def visitModule(self, mod): #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), long b) { return PyLong_FromLong(b); } @@ -1116,8 +1117,6 @@ def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) self.file.write(textwrap.dedent(''' - state->recursion_depth = 0; - state->recursion_limit = 0; return 0; } ''')) @@ -1260,7 +1259,7 @@ class ObjVisitor(PickleVisitor): def func_begin(self, name): ctype = get_c_type(name) self.emit("PyObject*", 0) - self.emit("ast2obj_%s(struct ast_state *state, void* _o)" % (name), 0) + self.emit("ast2obj_%s(struct ast_state *state, struct validator *vstate, void* _o)" % (name), 0) self.emit("{", 0) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("PyObject *result = NULL, *value = NULL;", 1) @@ -1268,17 +1267,17 @@ def func_begin(self, name): self.emit('if (!o) {', 1) self.emit("Py_RETURN_NONE;", 2) self.emit("}", 1) - self.emit("if (++state->recursion_depth > state->recursion_limit) {", 1) + self.emit("if (++vstate->recursion_depth > vstate->recursion_limit) {", 1) self.emit("PyErr_SetString(PyExc_RecursionError,", 2) self.emit('"maximum recursion depth exceeded during ast construction");', 3) self.emit("return NULL;", 2) self.emit("}", 1) def func_end(self): - self.emit("state->recursion_depth--;", 1) + self.emit("vstate->recursion_depth--;", 1) self.emit("return result;", 1) self.emit("failed:", 0) - self.emit("state->recursion_depth--;", 1) + self.emit("vstate->recursion_depth--;", 1) self.emit("Py_XDECREF(value);", 1) self.emit("Py_XDECREF(result);", 1) self.emit("return NULL;", 1) @@ -1296,7 +1295,7 @@ def visitSum(self, sum, name): self.visitConstructor(t, i + 1, name) self.emit("}", 1) for a in sum.attributes: - self.emit("value = ast2obj_%s(state, o->%s);" % (a.type, a.name), 1) + self.emit("value = ast2obj_%s(state, vstate, o->%s);" % (a.type, a.name), 1) self.emit("if (!value) goto failed;", 1) self.emit('if (PyObject_SetAttr(result, state->%s, value) < 0)' % a.name, 1) self.emit('goto failed;', 2) @@ -1304,7 +1303,7 @@ def visitSum(self, sum, name): self.func_end() def simpleSum(self, sum, name): - self.emit("PyObject* ast2obj_%s(struct ast_state *state, %s_ty o)" % (name, name), 0) + self.emit("PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, %s_ty o)" % (name, name), 0) self.emit("{", 0) self.emit("switch(o) {", 1) for t in sum.types: @@ -1322,7 +1321,7 @@ def visitProduct(self, prod, name): for field in prod.fields: self.visitField(field, name, 1, True) for a in prod.attributes: - self.emit("value = ast2obj_%s(state, o->%s);" % (a.type, a.name), 1) + self.emit("value = ast2obj_%s(state, vstate, o->%s);" % (a.type, a.name), 1) self.emit("if (!value) goto failed;", 1) self.emit("if (PyObject_SetAttr(result, state->%s, value) < 0)" % a.name, 1) self.emit('goto failed;', 2) @@ -1363,7 +1362,7 @@ def set(self, field, value, depth): self.emit("for(i = 0; i < n; i++)", depth+1) # This cannot fail, so no need for error handling self.emit( - "PyList_SET_ITEM(value, i, ast2obj_{0}(state, ({0}_ty)asdl_seq_GET({1}, i)));".format( + "PyList_SET_ITEM(value, i, ast2obj_{0}(state, vstate, ({0}_ty)asdl_seq_GET({1}, i)));".format( field.type, value ), @@ -1372,9 +1371,9 @@ def set(self, field, value, depth): ) self.emit("}", depth) else: - self.emit("value = ast2obj_list(state, (asdl_seq*)%s, ast2obj_%s);" % (value, field.type), depth) + self.emit("value = ast2obj_list(state, vstate, (asdl_seq*)%s, ast2obj_%s);" % (value, field.type), depth) else: - self.emit("value = ast2obj_%s(state, %s);" % (field.type, value), depth, reflow=False) + self.emit("value = ast2obj_%s(state, vstate, %s);" % (field.type, value), depth, reflow=False) class PartingShots(StaticVisitor): @@ -1394,18 +1393,19 @@ class PartingShots(StaticVisitor): if (!tstate) { return NULL; } - state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + struct validator vstate; + vstate.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; - state->recursion_depth = starting_recursion_depth; + vstate.recursion_depth = starting_recursion_depth; - PyObject *result = ast2obj_mod(state, t); + PyObject *result = ast2obj_mod(state, &vstate, t); /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { + if (result && vstate.recursion_depth != starting_recursion_depth) { PyErr_Format(PyExc_SystemError, "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); + starting_recursion_depth, vstate.recursion_depth); return NULL; } return result; @@ -1475,8 +1475,6 @@ def generate_ast_state(module_state, f): f.write('struct ast_state {\n') f.write(' _PyOnceFlag once;\n') f.write(' int finalized;\n') - f.write(' int recursion_depth;\n') - f.write(' int recursion_limit;\n') for s in module_state: f.write(' PyObject *' + s + ';\n') f.write('};') @@ -1539,6 +1537,11 @@ def generate_module_def(mod, metadata, f, internal_h): #include "pycore_pystate.h" // _PyInterpreterState_GET() #include + struct validator { + int recursion_depth; /* current recursion depth */ + int recursion_limit; /* recursion limit */ + }; + // Forward declaration static int init_types(struct ast_state *state); diff --git a/Parser/pegen.c b/Parser/pegen.c index 0c60394e4f199b..7766253a76066f 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -19,12 +19,8 @@ _PyPegen_interactive_exit(Parser *p) } Py_ssize_t -_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) +_PyPegen_byte_offset_to_character_offset_raw(const char* str, Py_ssize_t col_offset) { - const char *str = PyUnicode_AsUTF8(line); - if (!str) { - return -1; - } Py_ssize_t len = strlen(str); if (col_offset > len + 1) { col_offset = len + 1; @@ -39,6 +35,16 @@ _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) return size; } +Py_ssize_t +_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset) +{ + const char *str = PyUnicode_AsUTF8(line); + if (!str) { + return -1; + } + return _PyPegen_byte_offset_to_character_offset_raw(str, col_offset); +} + // Here, mark is the start of the node, while p->mark is the end. // If node==NULL, they should be the same. int diff --git a/Parser/pegen.h b/Parser/pegen.h index 424f80acd7be3b..57b45a54c36c57 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -149,6 +149,7 @@ expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset); +Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset); // Error handling functions and APIs typedef enum { diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index e2bc3b91c80718..e15673d02dd3b0 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -219,6 +219,10 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { void * _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...) { + // Bail out if we already have an error set. + if (p->error_indicator && PyErr_Occurred()) { + return NULL; + } if (p->fill == 0) { va_list va; va_start(va, errmsg); @@ -278,6 +282,10 @@ get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno) Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno; const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp; + if (buf_end < cur_line) { + buf_end = cur_line + strlen(cur_line); + } + for (int i = 0; i < relative_lineno - 1; i++) { char *new_line = strchr(cur_line, '\n'); // The assert is here for debug builds but the conditional that @@ -303,6 +311,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t end_lineno, Py_ssize_t end_col_offset, const char *errmsg, va_list va) { + // Bail out if we already have an error set. + if (p->error_indicator && PyErr_Occurred()) { + return NULL; + } PyObject *value = NULL; PyObject *errstr = NULL; PyObject *error_line = NULL; @@ -394,7 +406,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, void _Pypegen_set_syntax_error(Parser* p, Token* last_token) { - // Existing sintax error + // Existing syntax error if (PyErr_Occurred()) { // Prioritize tokenizer errors to custom syntax errors raised // on the second phase only if the errors come from the parser. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 1f9aa4b3d449a1..30998bf80f9ce4 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -576,7 +576,11 @@ static int test_init_from_config(void) _PyPreConfig_InitCompatConfig(&preconfig); putenv("PYTHONMALLOC=malloc_debug"); +#ifndef Py_GIL_DISABLED preconfig.allocator = PYMEM_ALLOCATOR_MALLOC; +#else + preconfig.allocator = PYMEM_ALLOCATOR_MIMALLOC; +#endif putenv("PYTHONUTF8=0"); Py_UTF8Mode = 0; @@ -765,7 +769,11 @@ static int test_init_dont_parse_argv(void) static void set_most_env_vars(void) { putenv("PYTHONHASHSEED=42"); +#ifndef Py_GIL_DISABLED putenv("PYTHONMALLOC=malloc"); +#else + putenv("PYTHONMALLOC=mimalloc"); +#endif putenv("PYTHONTRACEMALLOC=2"); putenv("PYTHONPROFILEIMPORTTIME=1"); putenv("PYTHONNODEBUGRANGES=1"); @@ -851,7 +859,11 @@ static int test_init_env_dev_mode_alloc(void) /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; set_all_env_vars_dev_mode(); +#ifndef Py_GIL_DISABLED putenv("PYTHONMALLOC=malloc"); +#else + putenv("PYTHONMALLOC=mimalloc"); +#endif _testembed_Py_InitializeFromConfig(); dump_config(); Py_Finalize(); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 75dc3e156aa945..699e1c157c591c 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -9,6 +9,11 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include +struct validator { + int recursion_depth; /* current recursion depth */ + int recursion_limit; /* recursion limit */ +}; + // Forward declaration static int init_types(struct ast_state *state); @@ -383,7 +388,8 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(pattern, pattern_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_param, type_param_ty) -static PyObject* ast2obj_mod(struct ast_state *state, void*); +static PyObject* ast2obj_mod(struct ast_state *state, struct validator *vstate, + void*); static const char * const Module_fields[]={ "body", "type_ignores", @@ -404,7 +410,8 @@ static const char * const stmt_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_stmt(struct ast_state *state, void*); +static PyObject* ast2obj_stmt(struct ast_state *state, struct validator + *vstate, void*); static const char * const FunctionDef_fields[]={ "name", "args", @@ -539,7 +546,8 @@ static const char * const expr_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_expr(struct ast_state *state, void*); +static PyObject* ast2obj_expr(struct ast_state *state, struct validator + *vstate, void*); static const char * const BoolOp_fields[]={ "op", "values", @@ -652,12 +660,18 @@ static const char * const Slice_fields[]={ "upper", "step", }; -static PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty); -static PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty); -static PyObject* ast2obj_operator(struct ast_state *state, operator_ty); -static PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty); -static PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty); -static PyObject* ast2obj_comprehension(struct ast_state *state, void*); +static PyObject* ast2obj_expr_context(struct ast_state *state, struct validator + *vstate, expr_context_ty); +static PyObject* ast2obj_boolop(struct ast_state *state, struct validator + *vstate, boolop_ty); +static PyObject* ast2obj_operator(struct ast_state *state, struct validator + *vstate, operator_ty); +static PyObject* ast2obj_unaryop(struct ast_state *state, struct validator + *vstate, unaryop_ty); +static PyObject* ast2obj_cmpop(struct ast_state *state, struct validator + *vstate, cmpop_ty); +static PyObject* ast2obj_comprehension(struct ast_state *state, struct + validator *vstate, void*); static const char * const comprehension_fields[]={ "target", "iter", @@ -670,13 +684,15 @@ static const char * const excepthandler_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_excepthandler(struct ast_state *state, void*); +static PyObject* ast2obj_excepthandler(struct ast_state *state, struct + validator *vstate, void*); static const char * const ExceptHandler_fields[]={ "type", "name", "body", }; -static PyObject* ast2obj_arguments(struct ast_state *state, void*); +static PyObject* ast2obj_arguments(struct ast_state *state, struct validator + *vstate, void*); static const char * const arguments_fields[]={ "posonlyargs", "args", @@ -686,7 +702,8 @@ static const char * const arguments_fields[]={ "kwarg", "defaults", }; -static PyObject* ast2obj_arg(struct ast_state *state, void*); +static PyObject* ast2obj_arg(struct ast_state *state, struct validator *vstate, + void*); static const char * const arg_attributes[] = { "lineno", "col_offset", @@ -698,7 +715,8 @@ static const char * const arg_fields[]={ "annotation", "type_comment", }; -static PyObject* ast2obj_keyword(struct ast_state *state, void*); +static PyObject* ast2obj_keyword(struct ast_state *state, struct validator + *vstate, void*); static const char * const keyword_attributes[] = { "lineno", "col_offset", @@ -709,7 +727,8 @@ static const char * const keyword_fields[]={ "arg", "value", }; -static PyObject* ast2obj_alias(struct ast_state *state, void*); +static PyObject* ast2obj_alias(struct ast_state *state, struct validator + *vstate, void*); static const char * const alias_attributes[] = { "lineno", "col_offset", @@ -720,12 +739,14 @@ static const char * const alias_fields[]={ "name", "asname", }; -static PyObject* ast2obj_withitem(struct ast_state *state, void*); +static PyObject* ast2obj_withitem(struct ast_state *state, struct validator + *vstate, void*); static const char * const withitem_fields[]={ "context_expr", "optional_vars", }; -static PyObject* ast2obj_match_case(struct ast_state *state, void*); +static PyObject* ast2obj_match_case(struct ast_state *state, struct validator + *vstate, void*); static const char * const match_case_fields[]={ "pattern", "guard", @@ -737,7 +758,8 @@ static const char * const pattern_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_pattern(struct ast_state *state, void*); +static PyObject* ast2obj_pattern(struct ast_state *state, struct validator + *vstate, void*); static const char * const MatchValue_fields[]={ "value", }; @@ -768,7 +790,8 @@ static const char * const MatchAs_fields[]={ static const char * const MatchOr_fields[]={ "patterns", }; -static PyObject* ast2obj_type_ignore(struct ast_state *state, void*); +static PyObject* ast2obj_type_ignore(struct ast_state *state, struct validator + *vstate, void*); static const char * const TypeIgnore_fields[]={ "lineno", "tag", @@ -779,7 +802,8 @@ static const char * const type_param_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_type_param(struct ast_state *state, void*); +static PyObject* ast2obj_type_param(struct ast_state *state, struct validator + *vstate, void*); static const char * const TypeVar_fields[]={ "name", "bound", @@ -1008,7 +1032,8 @@ add_attributes(struct ast_state *state, PyObject *type, const char * const *attr /* Conversion AST -> Python */ -static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, struct validator *vstate, asdl_seq *seq, + PyObject* (*func)(struct ast_state *state, struct validator *vstate, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -1016,7 +1041,7 @@ static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* if (!result) return NULL; for (i = 0; i < n; i++) { - value = func(state, asdl_seq_GET_UNTYPED(seq, i)); + value = func(state, vstate, asdl_seq_GET_UNTYPED(seq, i)); if (!value) { Py_DECREF(result); return NULL; @@ -1026,7 +1051,7 @@ static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* return result; } -static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), void *o) { PyObject *op = (PyObject*)o; if (!op) { @@ -1038,7 +1063,7 @@ static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), struct validator *Py_UNUSED(vstate), long b) { return PyLong_FromLong(b); } @@ -1914,8 +1939,6 @@ init_types(struct ast_state *state) "TypeVarTuple(identifier name)"); if (!state->TypeVarTuple_type) return -1; - state->recursion_depth = 0; - state->recursion_limit = 0; return 0; } @@ -3770,7 +3793,7 @@ _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int PyObject* -ast2obj_mod(struct ast_state *state, void* _o) +ast2obj_mod(struct ast_state *state, struct validator *vstate, void* _o) { mod_ty o = (mod_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3778,7 +3801,7 @@ ast2obj_mod(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -3788,12 +3811,14 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Module_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Module.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Module.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Module.type_ignores, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.Module.type_ignores, ast2obj_type_ignore); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_ignores, value) == -1) @@ -3804,7 +3829,7 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Interactive_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Interactive.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Interactive.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) @@ -3815,7 +3840,7 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Expression_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Expression.body); + value = ast2obj_expr(state, vstate, o->v.Expression.body); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; @@ -3825,30 +3850,31 @@ ast2obj_mod(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->FunctionType_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionType.argtypes, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.FunctionType.argtypes, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->argtypes, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.FunctionType.returns); + value = ast2obj_expr(state, vstate, o->v.FunctionType.returns); if (!value) goto failed; if (PyObject_SetAttr(result, state->returns, value) == -1) goto failed; Py_DECREF(value); break; } - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_stmt(struct ast_state *state, void* _o) +ast2obj_stmt(struct ast_state *state, struct validator *vstate, void* _o) { stmt_ty o = (stmt_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3856,7 +3882,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -3866,39 +3892,41 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->FunctionDef_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.FunctionDef.name); + value = ast2obj_identifier(state, vstate, o->v.FunctionDef.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arguments(state, o->v.FunctionDef.args); + value = ast2obj_arguments(state, vstate, o->v.FunctionDef.args); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.FunctionDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.decorator_list, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.FunctionDef.decorator_list, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->decorator_list, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.FunctionDef.returns); + value = ast2obj_expr(state, vstate, o->v.FunctionDef.returns); if (!value) goto failed; if (PyObject_SetAttr(result, state->returns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.FunctionDef.type_comment); + value = ast2obj_string(state, vstate, o->v.FunctionDef.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.type_params, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.FunctionDef.type_params, ast2obj_type_param); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_params, value) == -1) @@ -3909,40 +3937,41 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AsyncFunctionDef_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.AsyncFunctionDef.name); + value = ast2obj_identifier(state, vstate, o->v.AsyncFunctionDef.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arguments(state, o->v.AsyncFunctionDef.args); + value = ast2obj_arguments(state, vstate, o->v.AsyncFunctionDef.args); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncFunctionDef.body, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.AsyncFunctionDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFunctionDef.decorator_list, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->decorator_list, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AsyncFunctionDef.returns); + value = ast2obj_expr(state, vstate, o->v.AsyncFunctionDef.returns); if (!value) goto failed; if (PyObject_SetAttr(result, state->returns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.AsyncFunctionDef.type_comment); + value = ast2obj_string(state, vstate, o->v.AsyncFunctionDef.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFunctionDef.type_params, ast2obj_type_param); if (!value) goto failed; @@ -3954,36 +3983,38 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ClassDef_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.ClassDef.name); + value = ast2obj_identifier(state, vstate, o->v.ClassDef.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.bases, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ClassDef.bases, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->bases, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.keywords, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ClassDef.keywords, ast2obj_keyword); if (!value) goto failed; if (PyObject_SetAttr(result, state->keywords, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ClassDef.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.decorator_list, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.ClassDef.decorator_list, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->decorator_list, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.type_params, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.ClassDef.type_params, ast2obj_type_param); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_params, value) == -1) @@ -3994,7 +4025,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Return_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Return.value); + value = ast2obj_expr(state, vstate, o->v.Return.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4004,7 +4035,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Delete_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Delete.targets, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Delete.targets, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->targets, value) == -1) @@ -4015,18 +4046,18 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Assign_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Assign.targets, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Assign.targets, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->targets, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Assign.value); + value = ast2obj_expr(state, vstate, o->v.Assign.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.Assign.type_comment); + value = ast2obj_string(state, vstate, o->v.Assign.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4036,18 +4067,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TypeAlias_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.TypeAlias.name); + value = ast2obj_expr(state, vstate, o->v.TypeAlias.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TypeAlias.type_params, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.TypeAlias.type_params, ast2obj_type_param); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.TypeAlias.value); + value = ast2obj_expr(state, vstate, o->v.TypeAlias.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4057,17 +4089,17 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AugAssign_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.AugAssign.target); + value = ast2obj_expr(state, vstate, o->v.AugAssign.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_operator(state, o->v.AugAssign.op); + value = ast2obj_operator(state, vstate, o->v.AugAssign.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AugAssign.value); + value = ast2obj_expr(state, vstate, o->v.AugAssign.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4077,22 +4109,22 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AnnAssign_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.AnnAssign.target); + value = ast2obj_expr(state, vstate, o->v.AnnAssign.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AnnAssign.annotation); + value = ast2obj_expr(state, vstate, o->v.AnnAssign.annotation); if (!value) goto failed; if (PyObject_SetAttr(result, state->annotation, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AnnAssign.value); + value = ast2obj_expr(state, vstate, o->v.AnnAssign.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->v.AnnAssign.simple); + value = ast2obj_int(state, vstate, o->v.AnnAssign.simple); if (!value) goto failed; if (PyObject_SetAttr(result, state->simple, value) == -1) goto failed; @@ -4102,27 +4134,29 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->For_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.For.target); + value = ast2obj_expr(state, vstate, o->v.For.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.For.iter); + value = ast2obj_expr(state, vstate, o->v.For.iter); if (!value) goto failed; if (PyObject_SetAttr(result, state->iter, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.For.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.For.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.For.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.For.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.For.type_comment); + value = ast2obj_string(state, vstate, o->v.For.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4132,29 +4166,29 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AsyncFor_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.AsyncFor.target); + value = ast2obj_expr(state, vstate, o->v.AsyncFor.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.AsyncFor.iter); + value = ast2obj_expr(state, vstate, o->v.AsyncFor.iter); if (!value) goto failed; if (PyObject_SetAttr(result, state->iter, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncFor.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFor.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncFor.orelse, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncFor.orelse, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.AsyncFor.type_comment); + value = ast2obj_string(state, vstate, o->v.AsyncFor.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4164,17 +4198,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->While_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.While.test); + value = ast2obj_expr(state, vstate, o->v.While.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.While.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.While.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.While.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.While.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; @@ -4184,17 +4220,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->If_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.If.test); + value = ast2obj_expr(state, vstate, o->v.If.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.If.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.If.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.If.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.If.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; @@ -4204,18 +4242,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->With_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.With.items, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.With.items, ast2obj_withitem); if (!value) goto failed; if (PyObject_SetAttr(result, state->items, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.With.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.With.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.With.type_comment); + value = ast2obj_string(state, vstate, o->v.With.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4225,19 +4264,19 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->AsyncWith_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncWith.items, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncWith.items, ast2obj_withitem); if (!value) goto failed; if (PyObject_SetAttr(result, state->items, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.AsyncWith.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.AsyncWith.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.AsyncWith.type_comment); + value = ast2obj_string(state, vstate, o->v.AsyncWith.type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; @@ -4247,12 +4286,12 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Match_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Match.subject); + value = ast2obj_expr(state, vstate, o->v.Match.subject); if (!value) goto failed; if (PyObject_SetAttr(result, state->subject, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Match.cases, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Match.cases, ast2obj_match_case); if (!value) goto failed; if (PyObject_SetAttr(result, state->cases, value) == -1) @@ -4263,12 +4302,12 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Raise_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Raise.exc); + value = ast2obj_expr(state, vstate, o->v.Raise.exc); if (!value) goto failed; if (PyObject_SetAttr(result, state->exc, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Raise.cause); + value = ast2obj_expr(state, vstate, o->v.Raise.cause); if (!value) goto failed; if (PyObject_SetAttr(result, state->cause, value) == -1) goto failed; @@ -4278,23 +4317,25 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Try_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Try.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Try.handlers, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.handlers, ast2obj_excepthandler); if (!value) goto failed; if (PyObject_SetAttr(result, state->handlers, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Try.orelse, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.orelse, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Try.finalbody, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Try.finalbody, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->finalbody, value) == -1) @@ -4305,24 +4346,25 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TryStar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.body, + ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.handlers, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.handlers, ast2obj_excepthandler); if (!value) goto failed; if (PyObject_SetAttr(result, state->handlers, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.orelse, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.orelse, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TryStar.finalbody, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.TryStar.finalbody, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->finalbody, value) == -1) @@ -4333,12 +4375,12 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Assert_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Assert.test); + value = ast2obj_expr(state, vstate, o->v.Assert.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Assert.msg); + value = ast2obj_expr(state, vstate, o->v.Assert.msg); if (!value) goto failed; if (PyObject_SetAttr(result, state->msg, value) == -1) goto failed; @@ -4348,7 +4390,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Import_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Import.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Import.names, ast2obj_alias); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) @@ -4359,18 +4401,18 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ImportFrom_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.ImportFrom.module); + value = ast2obj_identifier(state, vstate, o->v.ImportFrom.module); if (!value) goto failed; if (PyObject_SetAttr(result, state->module, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ImportFrom.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ImportFrom.names, ast2obj_alias); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->v.ImportFrom.level); + value = ast2obj_int(state, vstate, o->v.ImportFrom.level); if (!value) goto failed; if (PyObject_SetAttr(result, state->level, value) == -1) goto failed; @@ -4380,7 +4422,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Global_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Global.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Global.names, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) @@ -4391,7 +4433,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Nonlocal_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Nonlocal.names, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Nonlocal.names, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttr(result, state->names, value) == -1) @@ -4402,7 +4444,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Expr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Expr.value); + value = ast2obj_expr(state, vstate, o->v.Expr.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4424,37 +4466,37 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (!result) goto failed; break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_expr(struct ast_state *state, void* _o) +ast2obj_expr(struct ast_state *state, struct validator *vstate, void* _o) { expr_ty o = (expr_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4462,7 +4504,7 @@ ast2obj_expr(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -4472,12 +4514,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->BoolOp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_boolop(state, o->v.BoolOp.op); + value = ast2obj_boolop(state, vstate, o->v.BoolOp.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.BoolOp.values, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.BoolOp.values, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->values, value) == -1) @@ -4488,12 +4530,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->NamedExpr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.NamedExpr.target); + value = ast2obj_expr(state, vstate, o->v.NamedExpr.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.NamedExpr.value); + value = ast2obj_expr(state, vstate, o->v.NamedExpr.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4503,17 +4545,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->BinOp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.BinOp.left); + value = ast2obj_expr(state, vstate, o->v.BinOp.left); if (!value) goto failed; if (PyObject_SetAttr(result, state->left, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_operator(state, o->v.BinOp.op); + value = ast2obj_operator(state, vstate, o->v.BinOp.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.BinOp.right); + value = ast2obj_expr(state, vstate, o->v.BinOp.right); if (!value) goto failed; if (PyObject_SetAttr(result, state->right, value) == -1) goto failed; @@ -4523,12 +4565,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->UnaryOp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_unaryop(state, o->v.UnaryOp.op); + value = ast2obj_unaryop(state, vstate, o->v.UnaryOp.op); if (!value) goto failed; if (PyObject_SetAttr(result, state->op, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.UnaryOp.operand); + value = ast2obj_expr(state, vstate, o->v.UnaryOp.operand); if (!value) goto failed; if (PyObject_SetAttr(result, state->operand, value) == -1) goto failed; @@ -4538,12 +4580,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Lambda_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_arguments(state, o->v.Lambda.args); + value = ast2obj_arguments(state, vstate, o->v.Lambda.args); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Lambda.body); + value = ast2obj_expr(state, vstate, o->v.Lambda.body); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; @@ -4553,17 +4595,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->IfExp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.IfExp.test); + value = ast2obj_expr(state, vstate, o->v.IfExp.test); if (!value) goto failed; if (PyObject_SetAttr(result, state->test, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.IfExp.body); + value = ast2obj_expr(state, vstate, o->v.IfExp.body); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.IfExp.orelse); + value = ast2obj_expr(state, vstate, o->v.IfExp.orelse); if (!value) goto failed; if (PyObject_SetAttr(result, state->orelse, value) == -1) goto failed; @@ -4573,12 +4615,14 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Dict_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Dict.keys, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Dict.keys, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->keys, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Dict.values, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Dict.values, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->values, value) == -1) goto failed; @@ -4588,7 +4632,8 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Set_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Set.elts, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Set.elts, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->elts, value) == -1) goto failed; @@ -4598,12 +4643,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ListComp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.ListComp.elt); + value = ast2obj_expr(state, vstate, o->v.ListComp.elt); if (!value) goto failed; if (PyObject_SetAttr(result, state->elt, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ListComp.generators, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.ListComp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4614,12 +4660,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->SetComp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.SetComp.elt); + value = ast2obj_expr(state, vstate, o->v.SetComp.elt); if (!value) goto failed; if (PyObject_SetAttr(result, state->elt, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.SetComp.generators, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.SetComp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4630,17 +4676,18 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->DictComp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.DictComp.key); + value = ast2obj_expr(state, vstate, o->v.DictComp.key); if (!value) goto failed; if (PyObject_SetAttr(result, state->key, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.DictComp.value); + value = ast2obj_expr(state, vstate, o->v.DictComp.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.DictComp.generators, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.DictComp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4651,12 +4698,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->GeneratorExp_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.GeneratorExp.elt); + value = ast2obj_expr(state, vstate, o->v.GeneratorExp.elt); if (!value) goto failed; if (PyObject_SetAttr(result, state->elt, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.GeneratorExp.generators, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.GeneratorExp.generators, ast2obj_comprehension); if (!value) goto failed; if (PyObject_SetAttr(result, state->generators, value) == -1) @@ -4667,7 +4715,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Await_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Await.value); + value = ast2obj_expr(state, vstate, o->v.Await.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4677,7 +4725,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Yield_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Yield.value); + value = ast2obj_expr(state, vstate, o->v.Yield.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4687,7 +4735,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->YieldFrom_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.YieldFrom.value); + value = ast2obj_expr(state, vstate, o->v.YieldFrom.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -4697,7 +4745,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Compare_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Compare.left); + value = ast2obj_expr(state, vstate, o->v.Compare.left); if (!value) goto failed; if (PyObject_SetAttr(result, state->left, value) == -1) goto failed; @@ -4707,14 +4755,14 @@ ast2obj_expr(struct ast_state *state, void* _o) value = PyList_New(n); if (!value) goto failed; for(i = 0; i < n; i++) - PyList_SET_ITEM(value, i, ast2obj_cmpop(state, (cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i))); + PyList_SET_ITEM(value, i, ast2obj_cmpop(state, vstate, (cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i))); } if (!value) goto failed; if (PyObject_SetAttr(result, state->ops, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Compare.comparators, - ast2obj_expr); + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.Compare.comparators, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->comparators, value) == -1) goto failed; @@ -4724,17 +4772,18 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Call_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Call.func); + value = ast2obj_expr(state, vstate, o->v.Call.func); if (!value) goto failed; if (PyObject_SetAttr(result, state->func, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Call.args, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Call.args, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.Call.keywords, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Call.keywords, ast2obj_keyword); if (!value) goto failed; if (PyObject_SetAttr(result, state->keywords, value) == -1) @@ -4745,17 +4794,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->FormattedValue_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.FormattedValue.value); + value = ast2obj_expr(state, vstate, o->v.FormattedValue.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->v.FormattedValue.conversion); + value = ast2obj_int(state, vstate, o->v.FormattedValue.conversion); if (!value) goto failed; if (PyObject_SetAttr(result, state->conversion, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.FormattedValue.format_spec); + value = ast2obj_expr(state, vstate, o->v.FormattedValue.format_spec); if (!value) goto failed; if (PyObject_SetAttr(result, state->format_spec, value) == -1) goto failed; @@ -4765,7 +4814,7 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->JoinedStr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.JoinedStr.values, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.JoinedStr.values, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->values, value) == -1) @@ -4776,12 +4825,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Constant_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_constant(state, o->v.Constant.value); + value = ast2obj_constant(state, vstate, o->v.Constant.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.Constant.kind); + value = ast2obj_string(state, vstate, o->v.Constant.kind); if (!value) goto failed; if (PyObject_SetAttr(result, state->kind, value) == -1) goto failed; @@ -4791,17 +4840,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Attribute_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Attribute.value); + value = ast2obj_expr(state, vstate, o->v.Attribute.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.Attribute.attr); + value = ast2obj_identifier(state, vstate, o->v.Attribute.attr); if (!value) goto failed; if (PyObject_SetAttr(result, state->attr, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Attribute.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Attribute.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4811,17 +4860,17 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Subscript_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Subscript.value); + value = ast2obj_expr(state, vstate, o->v.Subscript.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Subscript.slice); + value = ast2obj_expr(state, vstate, o->v.Subscript.slice); if (!value) goto failed; if (PyObject_SetAttr(result, state->slice, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Subscript.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Subscript.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4831,12 +4880,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Starred_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Starred.value); + value = ast2obj_expr(state, vstate, o->v.Starred.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Starred.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Starred.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4846,12 +4895,12 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Name_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.Name.id); + value = ast2obj_identifier(state, vstate, o->v.Name.id); if (!value) goto failed; if (PyObject_SetAttr(result, state->id, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Name.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Name.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4861,12 +4910,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->List_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.List.elts, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.List.elts, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->elts, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.List.ctx); + value = ast2obj_expr_context(state, vstate, o->v.List.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4876,12 +4926,13 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Tuple_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.Tuple.elts, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.Tuple.elts, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->elts, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(state, o->v.Tuple.ctx); + value = ast2obj_expr_context(state, vstate, o->v.Tuple.ctx); if (!value) goto failed; if (PyObject_SetAttr(result, state->ctx, value) == -1) goto failed; @@ -4891,53 +4942,54 @@ ast2obj_expr(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->Slice_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.Slice.lower); + value = ast2obj_expr(state, vstate, o->v.Slice.lower); if (!value) goto failed; if (PyObject_SetAttr(result, state->lower, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Slice.upper); + value = ast2obj_expr(state, vstate, o->v.Slice.upper); if (!value) goto failed; if (PyObject_SetAttr(result, state->upper, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.Slice.step); + value = ast2obj_expr(state, vstate, o->v.Slice.step); if (!value) goto failed; if (PyObject_SetAttr(result, state->step, value) == -1) goto failed; Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } -PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o) +PyObject* ast2obj_expr_context(struct ast_state *state, struct validator + *vstate, expr_context_ty o) { switch(o) { case Load: @@ -4949,7 +5001,8 @@ PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o) +PyObject* ast2obj_boolop(struct ast_state *state, struct validator *vstate, + boolop_ty o) { switch(o) { case And: @@ -4959,7 +5012,8 @@ PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_operator(struct ast_state *state, operator_ty o) +PyObject* ast2obj_operator(struct ast_state *state, struct validator *vstate, + operator_ty o) { switch(o) { case Add: @@ -4991,7 +5045,8 @@ PyObject* ast2obj_operator(struct ast_state *state, operator_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o) +PyObject* ast2obj_unaryop(struct ast_state *state, struct validator *vstate, + unaryop_ty o) { switch(o) { case Invert: @@ -5005,7 +5060,8 @@ PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o) +PyObject* ast2obj_cmpop(struct ast_state *state, struct validator *vstate, + cmpop_ty o) { switch(o) { case Eq: @@ -5032,7 +5088,8 @@ PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o) Py_UNREACHABLE(); } PyObject* -ast2obj_comprehension(struct ast_state *state, void* _o) +ast2obj_comprehension(struct ast_state *state, struct validator *vstate, void* + _o) { comprehension_ty o = (comprehension_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5040,7 +5097,7 @@ ast2obj_comprehension(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5048,37 +5105,38 @@ ast2obj_comprehension(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->comprehension_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_expr(state, o->target); + value = ast2obj_expr(state, vstate, o->target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->iter); + value = ast2obj_expr(state, vstate, o->iter); if (!value) goto failed; if (PyObject_SetAttr(result, state->iter, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->ifs, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->ifs, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->ifs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->is_async); + value = ast2obj_int(state, vstate, o->is_async); if (!value) goto failed; if (PyObject_SetAttr(result, state->is_async, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_excepthandler(struct ast_state *state, void* _o) +ast2obj_excepthandler(struct ast_state *state, struct validator *vstate, void* + _o) { excepthandler_ty o = (excepthandler_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5086,7 +5144,7 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5096,17 +5154,17 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ExceptHandler_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.ExceptHandler.type); + value = ast2obj_expr(state, vstate, o->v.ExceptHandler.type); if (!value) goto failed; if (PyObject_SetAttr(result, state->type, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.ExceptHandler.name); + value = ast2obj_identifier(state, vstate, o->v.ExceptHandler.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ExceptHandler.body, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.ExceptHandler.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) @@ -5114,37 +5172,37 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_arguments(struct ast_state *state, void* _o) +ast2obj_arguments(struct ast_state *state, struct validator *vstate, void* _o) { arguments_ty o = (arguments_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5152,7 +5210,7 @@ ast2obj_arguments(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5160,52 +5218,53 @@ ast2obj_arguments(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->arguments_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_list(state, (asdl_seq*)o->posonlyargs, ast2obj_arg); + value = ast2obj_list(state, vstate, (asdl_seq*)o->posonlyargs, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->posonlyargs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->args, ast2obj_arg); + value = ast2obj_list(state, vstate, (asdl_seq*)o->args, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->args, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arg(state, o->vararg); + value = ast2obj_arg(state, vstate, o->vararg); if (!value) goto failed; if (PyObject_SetAttr(result, state->vararg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->kwonlyargs, ast2obj_arg); + value = ast2obj_list(state, vstate, (asdl_seq*)o->kwonlyargs, ast2obj_arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwonlyargs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->kw_defaults, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->kw_defaults, + ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->kw_defaults, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arg(state, o->kwarg); + value = ast2obj_arg(state, vstate, o->kwarg); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwarg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->defaults, ast2obj_expr); + value = ast2obj_list(state, vstate, (asdl_seq*)o->defaults, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->defaults, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_arg(struct ast_state *state, void* _o) +ast2obj_arg(struct ast_state *state, struct validator *vstate, void* _o) { arg_ty o = (arg_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5213,7 +5272,7 @@ ast2obj_arg(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5221,52 +5280,52 @@ ast2obj_arg(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->arg_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(state, o->arg); + value = ast2obj_identifier(state, vstate, o->arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->arg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->annotation); + value = ast2obj_expr(state, vstate, o->annotation); if (!value) goto failed; if (PyObject_SetAttr(result, state->annotation, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->type_comment); + value = ast2obj_string(state, vstate, o->type_comment); if (!value) goto failed; if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_keyword(struct ast_state *state, void* _o) +ast2obj_keyword(struct ast_state *state, struct validator *vstate, void* _o) { keyword_ty o = (keyword_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5274,7 +5333,7 @@ ast2obj_keyword(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5282,47 +5341,47 @@ ast2obj_keyword(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->keyword_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(state, o->arg); + value = ast2obj_identifier(state, vstate, o->arg); if (!value) goto failed; if (PyObject_SetAttr(result, state->arg, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->value); + value = ast2obj_expr(state, vstate, o->value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_alias(struct ast_state *state, void* _o) +ast2obj_alias(struct ast_state *state, struct validator *vstate, void* _o) { alias_ty o = (alias_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5330,7 +5389,7 @@ ast2obj_alias(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5338,47 +5397,47 @@ ast2obj_alias(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->alias_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(state, o->name); + value = ast2obj_identifier(state, vstate, o->name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->asname); + value = ast2obj_identifier(state, vstate, o->asname); if (!value) goto failed; if (PyObject_SetAttr(result, state->asname, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_withitem(struct ast_state *state, void* _o) +ast2obj_withitem(struct ast_state *state, struct validator *vstate, void* _o) { withitem_ty o = (withitem_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5386,7 +5445,7 @@ ast2obj_withitem(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5394,27 +5453,27 @@ ast2obj_withitem(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->withitem_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_expr(state, o->context_expr); + value = ast2obj_expr(state, vstate, o->context_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->context_expr, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->optional_vars); + value = ast2obj_expr(state, vstate, o->optional_vars); if (!value) goto failed; if (PyObject_SetAttr(result, state->optional_vars, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_match_case(struct ast_state *state, void* _o) +ast2obj_match_case(struct ast_state *state, struct validator *vstate, void* _o) { match_case_ty o = (match_case_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5422,7 +5481,7 @@ ast2obj_match_case(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5430,32 +5489,32 @@ ast2obj_match_case(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->match_case_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_pattern(state, o->pattern); + value = ast2obj_pattern(state, vstate, o->pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->pattern, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->guard); + value = ast2obj_expr(state, vstate, o->guard); if (!value) goto failed; if (PyObject_SetAttr(result, state->guard, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->body, ast2obj_stmt); + value = ast2obj_list(state, vstate, (asdl_seq*)o->body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_pattern(struct ast_state *state, void* _o) +ast2obj_pattern(struct ast_state *state, struct validator *vstate, void* _o) { pattern_ty o = (pattern_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5463,7 +5522,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5473,7 +5532,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchValue_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.MatchValue.value); + value = ast2obj_expr(state, vstate, o->v.MatchValue.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -5483,7 +5542,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchSingleton_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_constant(state, o->v.MatchSingleton.value); + value = ast2obj_constant(state, vstate, o->v.MatchSingleton.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -5493,7 +5552,8 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchSequence_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchSequence.patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchSequence.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) @@ -5504,19 +5564,20 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchMapping_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchMapping.keys, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.MatchMapping.keys, ast2obj_expr); if (!value) goto failed; if (PyObject_SetAttr(result, state->keys, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchMapping.patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchMapping.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.MatchMapping.rest); + value = ast2obj_identifier(state, vstate, o->v.MatchMapping.rest); if (!value) goto failed; if (PyObject_SetAttr(result, state->rest, value) == -1) goto failed; @@ -5526,24 +5587,27 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchClass_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(state, o->v.MatchClass.cls); + value = ast2obj_expr(state, vstate, o->v.MatchClass.cls); if (!value) goto failed; if (PyObject_SetAttr(result, state->cls, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchClass.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.kwd_attrs, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchClass.kwd_attrs, ast2obj_identifier); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwd_attrs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.kwd_patterns, + value = ast2obj_list(state, vstate, + (asdl_seq*)o->v.MatchClass.kwd_patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->kwd_patterns, value) == -1) @@ -5554,7 +5618,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchStar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.MatchStar.name); + value = ast2obj_identifier(state, vstate, o->v.MatchStar.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; @@ -5564,12 +5628,12 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchAs_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_pattern(state, o->v.MatchAs.pattern); + value = ast2obj_pattern(state, vstate, o->v.MatchAs.pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->pattern, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.MatchAs.name); + value = ast2obj_identifier(state, vstate, o->v.MatchAs.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; @@ -5579,7 +5643,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchOr_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchOr.patterns, + value = ast2obj_list(state, vstate, (asdl_seq*)o->v.MatchOr.patterns, ast2obj_pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->patterns, value) == -1) @@ -5587,37 +5651,37 @@ ast2obj_pattern(struct ast_state *state, void* _o) Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_type_ignore(struct ast_state *state, void* _o) +ast2obj_type_ignore(struct ast_state *state, struct validator *vstate, void* _o) { type_ignore_ty o = (type_ignore_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5625,7 +5689,7 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5635,29 +5699,29 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TypeIgnore_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_int(state, o->v.TypeIgnore.lineno); + value = ast2obj_int(state, vstate, o->v.TypeIgnore.lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.TypeIgnore.tag); + value = ast2obj_string(state, vstate, o->v.TypeIgnore.tag); if (!value) goto failed; if (PyObject_SetAttr(result, state->tag, value) == -1) goto failed; Py_DECREF(value); break; } - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; } PyObject* -ast2obj_type_param(struct ast_state *state, void* _o) +ast2obj_type_param(struct ast_state *state, struct validator *vstate, void* _o) { type_param_ty o = (type_param_ty)_o; PyObject *result = NULL, *value = NULL; @@ -5665,7 +5729,7 @@ ast2obj_type_param(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { + if (++vstate->recursion_depth > vstate->recursion_limit) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded during ast construction"); return NULL; @@ -5675,12 +5739,12 @@ ast2obj_type_param(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TypeVar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.TypeVar.name); + value = ast2obj_identifier(state, vstate, o->v.TypeVar.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(state, o->v.TypeVar.bound); + value = ast2obj_expr(state, vstate, o->v.TypeVar.bound); if (!value) goto failed; if (PyObject_SetAttr(result, state->bound, value) == -1) goto failed; @@ -5690,7 +5754,7 @@ ast2obj_type_param(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->ParamSpec_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.ParamSpec.name); + value = ast2obj_identifier(state, vstate, o->v.ParamSpec.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; @@ -5700,37 +5764,37 @@ ast2obj_type_param(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->TypeVarTuple_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.TypeVarTuple.name); + value = ast2obj_identifier(state, vstate, o->v.TypeVarTuple.name); if (!value) goto failed; if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); break; } - value = ast2obj_int(state, o->lineno); + value = ast2obj_int(state, vstate, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->col_offset); + value = ast2obj_int(state, vstate, o->col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->col_offset, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_lineno); + value = ast2obj_int(state, vstate, o->end_lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_lineno, value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(state, o->end_col_offset); + value = ast2obj_int(state, vstate, o->end_col_offset); if (!value) goto failed; if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + vstate->recursion_depth--; return result; failed: - state->recursion_depth--; + vstate->recursion_depth--; Py_XDECREF(value); Py_XDECREF(result); return NULL; @@ -13090,18 +13154,19 @@ PyObject* PyAST_mod2obj(mod_ty t) if (!tstate) { return NULL; } - state->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + struct validator vstate; + vstate.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; - state->recursion_depth = starting_recursion_depth; + vstate.recursion_depth = starting_recursion_depth; - PyObject *result = ast2obj_mod(state, t); + PyObject *result = ast2obj_mod(state, &vstate, t); /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { + if (result && vstate.recursion_depth != starting_recursion_depth) { PyErr_Format(PyExc_SystemError, "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); + starting_recursion_depth, vstate.recursion_depth); return NULL; } return result; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 364fe55d0a05e4..a7891709b3b44a 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -225,7 +225,7 @@ tokenizeriter_next(tokenizeriterobject *it) col_offset = _PyPegen_byte_offset_to_character_offset(line, token.start - line_start); } if (token.end != NULL && token.end >= it->tok->line_start) { - end_col_offset = _PyPegen_byte_offset_to_character_offset(line, token.end - it->tok->line_start); + end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, token.end - it->tok->line_start); } if (it->tok->tok_extra_tokens) { diff --git a/Python/_warnings.c b/Python/_warnings.c index 4b7fb888247145..d4765032824e56 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -425,15 +425,15 @@ already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, Py_DECREF(version_obj); } else { - already_warned = PyDict_GetItemWithError(registry, key); + if (PyDict_GetItemRef(registry, key, &already_warned) < 0) { + return -1; + } if (already_warned != NULL) { int rc = PyObject_IsTrue(already_warned); + Py_DECREF(already_warned); if (rc != 0) return rc; } - else if (PyErr_Occurred()) { - return -1; - } } /* This warning wasn't found in the registry, set it. */ diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h deleted file mode 100644 index 0d7fbe8a39a5d4..00000000000000 --- a/Python/abstract_interp_cases.c.h +++ /dev/null @@ -1,966 +0,0 @@ -// This file is generated by Tools/cases_generator/generate_cases.py -// from: -// Python/bytecodes.c -// Do not edit! - - case NOP: { - break; - } - - case RESUME_CHECK: { - break; - } - - case POP_TOP: { - STACK_SHRINK(1); - break; - } - - case PUSH_NULL: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case END_SEND: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case UNARY_NEGATIVE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case UNARY_NOT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _TO_BOOL: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_BOOL: { - break; - } - - case TO_BOOL_INT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_LIST: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_NONE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_STR: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case TO_BOOL_ALWAYS_TRUE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case UNARY_INVERT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _GUARD_BOTH_INT: { - break; - } - - case _GUARD_BOTH_FLOAT: { - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _BINARY_OP_ADD_FLOAT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _GUARD_BOTH_UNICODE: { - break; - } - - case _BINARY_OP_ADD_UNICODE: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _BINARY_SUBSCR: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_SLICE: { - STACK_SHRINK(2); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case STORE_SLICE: { - STACK_SHRINK(4); - break; - } - - case BINARY_SUBSCR_LIST_INT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_SUBSCR_STR_INT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_SUBSCR_TUPLE_INT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BINARY_SUBSCR_DICT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LIST_APPEND: { - STACK_SHRINK(1); - break; - } - - case SET_ADD: { - STACK_SHRINK(1); - break; - } - - case _STORE_SUBSCR: { - STACK_SHRINK(3); - break; - } - - case STORE_SUBSCR_LIST_INT: { - STACK_SHRINK(3); - break; - } - - case STORE_SUBSCR_DICT: { - STACK_SHRINK(3); - break; - } - - case DELETE_SUBSCR: { - STACK_SHRINK(2); - break; - } - - case CALL_INTRINSIC_1: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_INTRINSIC_2: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _POP_FRAME: { - STACK_SHRINK(1); - break; - } - - case GET_AITER: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_ANEXT: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_AWAITABLE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case POP_EXCEPT: { - STACK_SHRINK(1); - break; - } - - case LOAD_ASSERTION_ERROR: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_BUILD_CLASS: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case STORE_NAME: { - STACK_SHRINK(1); - break; - } - - case DELETE_NAME: { - break; - } - - case _UNPACK_SEQUENCE: { - STACK_SHRINK(1); - STACK_GROW(oparg); - break; - } - - case UNPACK_SEQUENCE_TWO_TUPLE: { - STACK_SHRINK(1); - STACK_GROW(oparg); - break; - } - - case UNPACK_SEQUENCE_TUPLE: { - STACK_SHRINK(1); - STACK_GROW(oparg); - break; - } - - case UNPACK_SEQUENCE_LIST: { - STACK_SHRINK(1); - STACK_GROW(oparg); - break; - } - - case UNPACK_EX: { - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg >> 8))), true); - break; - } - - case _STORE_ATTR: { - STACK_SHRINK(2); - break; - } - - case DELETE_ATTR: { - STACK_SHRINK(1); - break; - } - - case STORE_GLOBAL: { - STACK_SHRINK(1); - break; - } - - case DELETE_GLOBAL: { - break; - } - - case LOAD_LOCALS: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_FROM_DICT_OR_GLOBALS: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_NAME: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _LOAD_GLOBAL: { - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _GUARD_GLOBALS_VERSION: { - break; - } - - case _GUARD_BUILTINS_VERSION: { - break; - } - - case _LOAD_GLOBAL_MODULE: { - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _LOAD_GLOBAL_BUILTINS: { - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case DELETE_FAST: { - break; - } - - case MAKE_CELL: { - break; - } - - case DELETE_DEREF: { - break; - } - - case LOAD_FROM_DICT_OR_DEREF: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LOAD_DEREF: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case STORE_DEREF: { - STACK_SHRINK(1); - break; - } - - case COPY_FREE_VARS: { - break; - } - - case BUILD_STRING: { - STACK_SHRINK(oparg); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BUILD_TUPLE: { - STACK_SHRINK(oparg); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BUILD_LIST: { - STACK_SHRINK(oparg); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case LIST_EXTEND: { - STACK_SHRINK(1); - break; - } - - case SET_UPDATE: { - STACK_SHRINK(1); - break; - } - - case BUILD_SET: { - STACK_SHRINK(oparg); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BUILD_MAP: { - STACK_SHRINK(oparg*2); - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case SETUP_ANNOTATIONS: { - break; - } - - case BUILD_CONST_KEY_MAP: { - STACK_SHRINK(oparg); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case DICT_UPDATE: { - STACK_SHRINK(1); - break; - } - - case DICT_MERGE: { - STACK_SHRINK(1); - break; - } - - case MAP_ADD: { - STACK_SHRINK(2); - break; - } - - case LOAD_SUPER_ATTR_ATTR: { - STACK_SHRINK(2); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); - break; - } - - case LOAD_SUPER_ATTR_METHOD: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _LOAD_ATTR: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _GUARD_TYPE_VERSION: { - break; - } - - case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - break; - } - - case _LOAD_ATTR_INSTANCE_VALUE: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _CHECK_ATTR_MODULE: { - break; - } - - case _LOAD_ATTR_MODULE: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _CHECK_ATTR_WITH_HINT: { - break; - } - - case _LOAD_ATTR_WITH_HINT: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _LOAD_ATTR_SLOT: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _CHECK_ATTR_CLASS: { - break; - } - - case _LOAD_ATTR_CLASS: { - STACK_GROW(((oparg & 1) ? 1 : 0)); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); - break; - } - - case _GUARD_DORV_VALUES: { - break; - } - - case _STORE_ATTR_INSTANCE_VALUE: { - STACK_SHRINK(2); - break; - } - - case _STORE_ATTR_SLOT: { - STACK_SHRINK(2); - break; - } - - case _COMPARE_OP: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case COMPARE_OP_FLOAT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case COMPARE_OP_INT: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case COMPARE_OP_STR: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case IS_OP: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CONTAINS_OP: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CHECK_EG_MATCH: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CHECK_EXC_MATCH: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _IS_NONE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_LEN: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MATCH_CLASS: { - STACK_SHRINK(2); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MATCH_MAPPING: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MATCH_SEQUENCE: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MATCH_KEYS: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_ITER: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case GET_YIELD_FROM_ITER: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _FOR_ITER_TIER_TWO: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _ITER_CHECK_LIST: { - break; - } - - case _GUARD_NOT_EXHAUSTED_LIST: { - break; - } - - case _ITER_NEXT_LIST: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _ITER_CHECK_TUPLE: { - break; - } - - case _GUARD_NOT_EXHAUSTED_TUPLE: { - break; - } - - case _ITER_NEXT_TUPLE: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _ITER_CHECK_RANGE: { - break; - } - - case _GUARD_NOT_EXHAUSTED_RANGE: { - break; - } - - case _ITER_NEXT_RANGE: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BEFORE_ASYNC_WITH: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BEFORE_WITH: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case WITH_EXCEPT_START: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case PUSH_EXC_INFO: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { - break; - } - - case _GUARD_KEYS_VERSION: { - break; - } - - case _LOAD_ATTR_METHOD_WITH_VALUES: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _LOAD_ATTR_METHOD_NO_DICT: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); - break; - } - - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); - break; - } - - case _CHECK_ATTR_METHOD_LAZY_DICT: { - break; - } - - case _LOAD_ATTR_METHOD_LAZY_DICT: { - STACK_GROW(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - break; - } - - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2 - oparg)), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - oparg)), true); - break; - } - - case _CHECK_PEP_523: { - break; - } - - case _CHECK_FUNCTION_EXACT_ARGS: { - break; - } - - case _CHECK_STACK_SPACE: { - break; - } - - case _INIT_CALL_PY_EXACT_ARGS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _PUSH_FRAME: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_TYPE_1: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_STR_1: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_TUPLE_1: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case EXIT_INIT_CHECK: { - STACK_SHRINK(1); - break; - } - - case CALL_BUILTIN_CLASS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_BUILTIN_O: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_BUILTIN_FAST: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_BUILTIN_FAST_WITH_KEYWORDS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_LEN: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_ISINSTANCE: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_METHOD_DESCRIPTOR_O: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_METHOD_DESCRIPTOR_NOARGS: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CALL_METHOD_DESCRIPTOR_FAST: { - STACK_SHRINK(oparg); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case MAKE_FUNCTION: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case SET_FUNCTION_ATTRIBUTE: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case BUILD_SLICE: { - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case CONVERT_VALUE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case FORMAT_SIMPLE: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case FORMAT_WITH_SPEC: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _BINARY_OP: { - STACK_SHRINK(1); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case SWAP: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2 - (oparg-2))), true); - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); - break; - } - - case _GUARD_IS_TRUE_POP: { - STACK_SHRINK(1); - break; - } - - case _GUARD_IS_FALSE_POP: { - STACK_SHRINK(1); - break; - } - - case _GUARD_IS_NONE_POP: { - STACK_SHRINK(1); - break; - } - - case _GUARD_IS_NOT_NONE_POP: { - STACK_SHRINK(1); - break; - } - - case _JUMP_TO_TOP: { - break; - } - - case _SET_IP: { - break; - } - - case _SAVE_RETURN_OFFSET: { - break; - } - - case _EXIT_TRACE: { - break; - } - - case _INSERT: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - oparg)), true); - break; - } - - case _CHECK_VALIDITY: { - break; - } diff --git a/Python/assemble.c b/Python/assemble.c index b6fb432aed4a3b..569454ebf3b9cb 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -4,7 +4,7 @@ #include "pycore_code.h" // write_location_entry_start() #include "pycore_compile.h" #include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE -#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR, _PyOpcode_Caches +#include "pycore_opcode_metadata.h" // is_pseudo_target, _PyOpcode_Caches #define DEFAULT_CODE_SIZE 128 @@ -710,13 +710,13 @@ resolve_unconditional_jumps(instr_sequence *instrs) bool is_forward = (instr->i_oparg > i); switch(instr->i_opcode) { case JUMP: - assert(SAME_OPCODE_METADATA(JUMP, JUMP_FORWARD)); - assert(SAME_OPCODE_METADATA(JUMP, JUMP_BACKWARD)); + assert(is_pseudo_target(JUMP, JUMP_FORWARD)); + assert(is_pseudo_target(JUMP, JUMP_BACKWARD)); instr->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD; break; case JUMP_NO_INTERRUPT: - assert(SAME_OPCODE_METADATA(JUMP_NO_INTERRUPT, JUMP_FORWARD)); - assert(SAME_OPCODE_METADATA(JUMP_NO_INTERRUPT, JUMP_BACKWARD_NO_INTERRUPT)); + assert(is_pseudo_target(JUMP_NO_INTERRUPT, JUMP_FORWARD)); + assert(is_pseudo_target(JUMP_NO_INTERRUPT, JUMP_BACKWARD_NO_INTERRUPT)); instr->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD_NO_INTERRUPT; break; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ff07c498263cd3..e54d5cbacdc96f 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -5,7 +5,6 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_Vector() #include "pycore_compile.h" // _PyAST_Compile() -#include "pycore_dict.h" // _PyDict_GetItemWithError() #include "pycore_long.h" // _PyLong_CompactValue #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _Py_AddToAllObjects() @@ -141,18 +140,16 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto error; } - meta = _PyDict_GetItemWithError(mkw, &_Py_ID(metaclass)); + if (PyDict_GetItemRef(mkw, &_Py_ID(metaclass), &meta) < 0) { + goto error; + } if (meta != NULL) { - Py_INCREF(meta); if (PyDict_DelItem(mkw, &_Py_ID(metaclass)) < 0) { goto error; } /* metaclass is explicitly given, check if it's indeed a class */ isclass = PyType_Check(meta); } - else if (PyErr_Occurred()) { - goto error; - } } if (meta == NULL) { /* if there are no bases, use type: */ @@ -1769,35 +1766,27 @@ builtin_locals_impl(PyObject *module) static PyObject * -min_max(PyObject *args, PyObject *kwds, int op) +min_max(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, int op) { - PyObject *v, *it, *item, *val, *maxitem, *maxval, *keyfunc=NULL; - PyObject *emptytuple, *defaultval = NULL; - static char *kwlist[] = {"key", "default", NULL}; - const char *name = op == Py_LT ? "min" : "max"; - const int positional = PyTuple_Size(args) > 1; - int ret; + PyObject *it = NULL, *item, *val, *maxitem, *maxval, *keyfunc=NULL; + PyObject *defaultval = NULL; + static const char * const keywords[] = {"key", "default", NULL}; + static _PyArg_Parser _parser_min = {"|$OO:min", keywords, 0}; + static _PyArg_Parser _parser_max = {"|$OO:max", keywords, 0}; + const char *name = (op == Py_LT) ? "min" : "max"; + _PyArg_Parser *_parser = (op == Py_LT) ? &_parser_min : &_parser_max; - if (positional) { - v = args; - } - else if (!PyArg_UnpackTuple(args, name, 1, 1, &v)) { - if (PyExceptionClass_Check(PyExc_TypeError)) { - PyErr_Format(PyExc_TypeError, "%s expected at least 1 argument, got 0", name); - } + if (nargs == 0) { + PyErr_Format(PyExc_TypeError, "%s expected at least 1 argument, got 0", name); return NULL; } - emptytuple = PyTuple_New(0); - if (emptytuple == NULL) - return NULL; - ret = PyArg_ParseTupleAndKeywords(emptytuple, kwds, - (op == Py_LT) ? "|$OO:min" : "|$OO:max", - kwlist, &keyfunc, &defaultval); - Py_DECREF(emptytuple); - if (!ret) + if (kwnames != NULL && !_PyArg_ParseStackAndKeywords(args + nargs, 0, kwnames, _parser, + &keyfunc, &defaultval)) { return NULL; + } + const int positional = nargs > 1; // False iff nargs == 1 if (positional && defaultval != NULL) { PyErr_Format(PyExc_TypeError, "Cannot specify a default for %s() with multiple " @@ -1805,9 +1794,11 @@ min_max(PyObject *args, PyObject *kwds, int op) return NULL; } - it = PyObject_GetIter(v); - if (it == NULL) { - return NULL; + if (!positional) { + it = PyObject_GetIter(args[0]); + if (it == NULL) { + return NULL; + } } if (keyfunc == Py_None) { @@ -1816,7 +1807,24 @@ min_max(PyObject *args, PyObject *kwds, int op) maxitem = NULL; /* the result */ maxval = NULL; /* the value associated with the result */ - while (( item = PyIter_Next(it) )) { + while (1) { + if (it == NULL) { + if (nargs-- <= 0) { + break; + } + item = *args++; + Py_INCREF(item); + } + else { + item = PyIter_Next(it); + if (item == NULL) { + if (PyErr_Occurred()) { + goto Fail_it; + } + break; + } + } + /* get the value from the key function */ if (keyfunc != NULL) { val = PyObject_CallOneArg(keyfunc, item); @@ -1850,8 +1858,6 @@ min_max(PyObject *args, PyObject *kwds, int op) } } } - if (PyErr_Occurred()) - goto Fail_it; if (maxval == NULL) { assert(maxitem == NULL); if (defaultval != NULL) { @@ -1863,7 +1869,7 @@ min_max(PyObject *args, PyObject *kwds, int op) } else Py_DECREF(maxval); - Py_DECREF(it); + Py_XDECREF(it); return maxitem; Fail_it_item_and_val: @@ -1873,15 +1879,15 @@ min_max(PyObject *args, PyObject *kwds, int op) Fail_it: Py_XDECREF(maxval); Py_XDECREF(maxitem); - Py_DECREF(it); + Py_XDECREF(it); return NULL; } /* AC: cannot convert yet, waiting for *args support */ static PyObject * -builtin_min(PyObject *self, PyObject *args, PyObject *kwds) +builtin_min(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return min_max(args, kwds, Py_LT); + return min_max(args, nargs, kwnames, Py_LT); } PyDoc_STRVAR(min_doc, @@ -1896,9 +1902,9 @@ With two or more positional arguments, return the smallest argument."); /* AC: cannot convert yet, waiting for *args support */ static PyObject * -builtin_max(PyObject *self, PyObject *args, PyObject *kwds) +builtin_max(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return min_max(args, kwds, Py_GT); + return min_max(args, nargs, kwnames, Py_GT); } PyDoc_STRVAR(max_doc, @@ -2265,6 +2271,11 @@ builtin_input_impl(PyObject *module, PyObject *prompt) goto _readline_errors; assert(PyBytes_Check(po)); promptstr = PyBytes_AS_STRING(po); + if ((Py_ssize_t)strlen(promptstr) != PyBytes_GET_SIZE(po)) { + PyErr_SetString(PyExc_ValueError, + "input: prompt string cannot contain null characters"); + goto _readline_errors; + } } else { po = NULL; @@ -3052,8 +3063,8 @@ static PyMethodDef builtin_methods[] = { BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF - {"max", _PyCFunction_CAST(builtin_max), METH_VARARGS | METH_KEYWORDS, max_doc}, - {"min", _PyCFunction_CAST(builtin_min), METH_VARARGS | METH_KEYWORDS, min_doc}, + {"max", _PyCFunction_CAST(builtin_max), METH_FASTCALL | METH_KEYWORDS, max_doc}, + {"min", _PyCFunction_CAST(builtin_min), METH_FASTCALL | METH_KEYWORDS, min_doc}, {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, next_doc}, BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a1ca66e457e47d..e1a6a256fbdf96 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -330,14 +330,14 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_TO_BOOL, (unused/2, value -- res)) { + op(_TO_BOOL, (value -- res)) { int err = PyObject_IsTrue(value); DECREF_INPUTS(); ERROR_IF(err < 0, error); res = err ? Py_True : Py_False; } - macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + _TO_BOOL; + macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL; inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { DEOPT_IF(!PyBool_Check(value)); @@ -416,7 +416,7 @@ dummy_func( DEOPT_IF(!PyLong_CheckExact(right)); } - op(_BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- res)) { + op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -424,7 +424,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - op(_BINARY_OP_ADD_INT, (unused/1, left, right -- res)) { + op(_BINARY_OP_ADD_INT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -432,7 +432,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - op(_BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- res)) { + op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -441,18 +441,18 @@ dummy_func( } macro(BINARY_OP_MULTIPLY_INT) = - _GUARD_BOTH_INT + _BINARY_OP_MULTIPLY_INT; + _GUARD_BOTH_INT + unused/1 + _BINARY_OP_MULTIPLY_INT; macro(BINARY_OP_ADD_INT) = - _GUARD_BOTH_INT + _BINARY_OP_ADD_INT; + _GUARD_BOTH_INT + unused/1 + _BINARY_OP_ADD_INT; macro(BINARY_OP_SUBTRACT_INT) = - _GUARD_BOTH_INT + _BINARY_OP_SUBTRACT_INT; + _GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { DEOPT_IF(!PyFloat_CheckExact(left)); DEOPT_IF(!PyFloat_CheckExact(right)); } - op(_BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- res)) { + op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * @@ -460,7 +460,7 @@ dummy_func( DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - op(_BINARY_OP_ADD_FLOAT, (unused/1, left, right -- res)) { + op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + @@ -468,7 +468,7 @@ dummy_func( DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - op(_BINARY_OP_SUBTRACT_FLOAT, (unused/1, left, right -- res)) { + op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - @@ -477,18 +477,18 @@ dummy_func( } macro(BINARY_OP_MULTIPLY_FLOAT) = - _GUARD_BOTH_FLOAT + _BINARY_OP_MULTIPLY_FLOAT; + _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_MULTIPLY_FLOAT; macro(BINARY_OP_ADD_FLOAT) = - _GUARD_BOTH_FLOAT + _BINARY_OP_ADD_FLOAT; + _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_ADD_FLOAT; macro(BINARY_OP_SUBTRACT_FLOAT) = - _GUARD_BOTH_FLOAT + _BINARY_OP_SUBTRACT_FLOAT; + _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { DEOPT_IF(!PyUnicode_CheckExact(left)); DEOPT_IF(!PyUnicode_CheckExact(right)); } - op(_BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { + op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -497,7 +497,7 @@ dummy_func( } macro(BINARY_OP_ADD_UNICODE) = - _GUARD_BOTH_UNICODE + _BINARY_OP_ADD_UNICODE; + _GUARD_BOTH_UNICODE + unused/1 + _BINARY_OP_ADD_UNICODE; // This is a subtle one. It's a super-instruction for // BINARY_OP_ADD_UNICODE followed by STORE_FAST @@ -505,7 +505,8 @@ dummy_func( // So the inputs are the same as for all BINARY_OP // specializations, but there is no output. // At the end we just skip over the STORE_FAST. - op(_BINARY_OP_INPLACE_ADD_UNICODE, (unused/1, left, right --)) { + op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { + TIER_ONE_ONLY assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left); @@ -532,7 +533,7 @@ dummy_func( } macro(BINARY_OP_INPLACE_ADD_UNICODE) = - _GUARD_BOTH_UNICODE + _BINARY_OP_INPLACE_ADD_UNICODE; + _GUARD_BOTH_UNICODE + unused/1 + _BINARY_OP_INPLACE_ADD_UNICODE; family(BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR) = { BINARY_SUBSCR_DICT, @@ -786,6 +787,7 @@ dummy_func( } inst(INTERPRETER_EXIT, (retval --)) { + TIER_ONE_ONLY assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ @@ -800,11 +802,11 @@ dummy_func( // We also push it onto the stack on exit, but that's a // different frame, and it's accounted for by _PUSH_FRAME. op(_POP_FRAME, (retval --)) { - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif STORE_SP(); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -1072,6 +1074,7 @@ dummy_func( } inst(YIELD_VALUE, (retval -- unused)) { + TIER_ONE_ONLY // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1097,7 +1100,7 @@ dummy_func( inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); + Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); } inst(RERAISE, (values[oparg], exc -- values[oparg])) { @@ -1165,7 +1168,6 @@ dummy_func( } } - inst(STORE_NAME, (v -- )) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -1293,14 +1295,14 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_STORE_ATTR, (unused/3, v, owner --)) { + op(_STORE_ATTR, (v, owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); DECREF_INPUTS(); ERROR_IF(err, error); } - macro(STORE_ATTR) = _SPECIALIZE_STORE_ATTR + _STORE_ATTR; + macro(STORE_ATTR) = _SPECIALIZE_STORE_ATTR + unused/3 + _STORE_ATTR; inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1412,7 +1414,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_LOAD_GLOBAL, (unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) { + op(_LOAD_GLOBAL, ( -- res, null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -1449,7 +1451,12 @@ dummy_func( null = NULL; } - macro(LOAD_GLOBAL) = _SPECIALIZE_LOAD_GLOBAL + _LOAD_GLOBAL; + macro(LOAD_GLOBAL) = + _SPECIALIZE_LOAD_GLOBAL + + counter/1 + + globals_version/1 + + builtins_version/1 + + _LOAD_GLOBAL; op(_GUARD_GLOBALS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1851,7 +1858,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_LOAD_ATTR, (unused/8, owner -- attr, self_or_null if (oparg & 1))) { + op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ @@ -1884,7 +1891,10 @@ dummy_func( } } - macro(LOAD_ATTR) = _SPECIALIZE_LOAD_ATTR + _LOAD_ATTR; + macro(LOAD_ATTR) = + _SPECIALIZE_LOAD_ATTR + + unused/8 + + _LOAD_ATTR; pseudo(LOAD_METHOD) = { LOAD_ATTR, @@ -2298,6 +2308,7 @@ dummy_func( } inst(JUMP_FORWARD, (--)) { + TIER_ONE_ONLY JUMPBY(oparg); } @@ -2353,23 +2364,31 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); - frame->instr_ptr = next_instr; - Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { - current_executor = (_PyUOpExecutorObject *)executor; - GOTO_TIER_TWO(); - } - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { + if (executor->vm_data.valid) { + Py_INCREF(executor); + if (executor->execute == _PyUOpExecute) { + current_executor = (_PyUOpExecutorObject *)executor; + GOTO_TIER_TWO(); + } + next_instr = executor->execute(executor, frame, stack_pointer); frame = tstate->current_frame; - goto resume_with_error; + if (next_instr == NULL) { + goto resume_with_error; + } + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + code->co_executors->executors[oparg & 255] = NULL; + opcode = this_instr->op.code = executor->vm_data.opcode; + this_instr->op.arg = executor->vm_data.oparg; + oparg = (oparg & (~255)) | executor->vm_data.oparg; + Py_DECREF(executor); + next_instr = this_instr; + DISPATCH_GOTO(); } - goto enter_tier_one; } - replaced op(_POP_JUMP_IF_FALSE, (unused/1, cond -- )) { + replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION @@ -2378,7 +2397,7 @@ dummy_func( JUMPBY(oparg * flag); } - replaced op(_POP_JUMP_IF_TRUE, (unused/1, cond -- )) { + replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION @@ -2397,15 +2416,16 @@ dummy_func( } } - macro(POP_JUMP_IF_TRUE) = _POP_JUMP_IF_TRUE; + macro(POP_JUMP_IF_TRUE) = unused/1 + _POP_JUMP_IF_TRUE; - macro(POP_JUMP_IF_FALSE) = _POP_JUMP_IF_FALSE; + macro(POP_JUMP_IF_FALSE) = unused/1 + _POP_JUMP_IF_FALSE; - macro(POP_JUMP_IF_NONE) = _IS_NONE + _POP_JUMP_IF_TRUE; + macro(POP_JUMP_IF_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_TRUE; - macro(POP_JUMP_IF_NOT_NONE) = _IS_NONE + _POP_JUMP_IF_FALSE; + macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + TIER_ONE_ONLY /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -2831,15 +2851,15 @@ dummy_func( ERROR_IF(res == NULL, error); } - pseudo(SETUP_FINALLY) = { + pseudo(SETUP_FINALLY, (HAS_ARG)) = { NOP, }; - pseudo(SETUP_CLEANUP) = { + pseudo(SETUP_CLEANUP, (HAS_ARG)) = { NOP, }; - pseudo(SETUP_WITH) = { + pseudo(SETUP_WITH, (HAS_ARG)) = { NOP, }; @@ -3009,7 +3029,7 @@ dummy_func( } // When calling Python, inline the call using DISPATCH_INLINED(). - op(_CALL, (unused/2, callable, self_or_null, args[oparg] -- res)) { + op(_CALL, (callable, self_or_null, args[oparg] -- res)) { // oparg counts all of the args, but *not* self: int total_args = oparg; if (self_or_null != NULL) { @@ -3078,7 +3098,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - macro(CALL) = _SPECIALIZE_CALL + _CALL; + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL; op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { DEOPT_IF(null != NULL); @@ -3130,7 +3150,7 @@ dummy_func( // The 'unused' output effect represents the return value // (which will be pushed when the frame returns). // It is needed so CALL_PY_EXACT_ARGS matches its family. - op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) { + op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused if (0))) { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); @@ -3458,6 +3478,7 @@ dummy_func( // This is secretly a super-instruction inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { + TIER_ONE_ONLY assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append); @@ -3796,6 +3817,7 @@ dummy_func( } inst(RETURN_GENERATOR, (--)) { + TIER_ONE_ONLY assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -3968,6 +3990,7 @@ dummy_func( } inst(EXTENDED_ARG, ( -- )) { + TIER_ONE_ONLY assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -3976,11 +3999,13 @@ dummy_func( } inst(CACHE, (--)) { + TIER_ONE_ONLY assert(0 && "Executing a cache."); Py_UNREACHABLE(); } inst(RESERVED, (--)) { + TIER_ONE_ONLY assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } @@ -4028,7 +4053,7 @@ dummy_func( op(_EXIT_TRACE, (--)) { TIER_TWO_ONLY - GOTO_TIER_ONE(); + DEOPT_IF(1); } op(_INSERT, (unused[oparg], top -- top, unused[oparg])) { diff --git a/Python/ceval.c b/Python/ceval.c index 76ab5df42f63db..1fea9747488102 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -24,6 +24,7 @@ #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() +#include "pycore_uop_ids.h" // Uops #include "pycore_uops.h" // _PyUOpExecutorObject #include "pycore_pyerrors.h" @@ -135,14 +136,14 @@ dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) static void lltrace_instruction(_PyInterpreterFrame *frame, PyObject **stack_pointer, - _Py_CODEUNIT *next_instr) + _Py_CODEUNIT *next_instr, + int opcode, + int oparg) { if (frame->owner == FRAME_OWNED_BY_CSTACK) { return; } dump_stack(frame, stack_pointer); - int oparg = next_instr->op.arg; - int opcode = next_instr->op.code; const char *opname = _PyOpcode_OpName[opcode]; assert(opname != NULL); int offset = (int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame))); @@ -647,7 +648,7 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { extern const struct _PyCode_DEF(8) _Py_InitCleanup; -extern const char *_PyUopName(int index); +extern const char *_PyUOpName(int index); /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ @@ -1002,7 +1003,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DPRINTF(3, "%4d: uop %s, oparg %d, operand %" PRIu64 ", target %d, stack_level %d\n", (int)(next_uop - current_executor->trace), - _PyUopName(uopcode), + _PyUOpName(uopcode), next_uop->oparg, next_uop->operand, next_uop->target, @@ -1051,9 +1052,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int pop_1_error_tier_two: STACK_SHRINK(1); error_tier_two: - DPRINTF(2, "Error: [Uop %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", - uopcode, _PyUopName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, - (int)(next_uop - current_executor->trace - 1)); + DPRINTF(2, "Error: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1062,30 +1064,16 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from DEOPT_IF() deoptimize: - // On DEOPT_IF we just repeat the last instruction. - // This presumes nothing was popped from the stack (nor pushed). - DPRINTF(2, "DEOPT: [Uop %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d]\n", - uopcode, _PyUopName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, - (int)(next_uop - current_executor->trace - 1)); + next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); + DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); - frame->return_offset = 0; // Dispatch to frame->instr_ptr - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); Py_DECREF(current_executor); - // Fall through -// Jump here from ENTER_EXECUTOR -enter_tier_one: - next_instr = frame->instr_ptr; - goto resume_frame; + DISPATCH(); -// Jump here from _EXIT_TRACE -exit_trace: - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); - Py_DECREF(current_executor); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - goto enter_tier_one; } #if defined(__GNUC__) # pragma GCC diagnostic pop @@ -1920,6 +1908,13 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) fixed_cause = _PyObject_CallNoArgs(cause); if (fixed_cause == NULL) goto raise_error; + if (!PyExceptionInstance_Check(fixed_cause)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + cause, Py_TYPE(fixed_cause)); + goto raise_error; + } Py_DECREF(cause); } else if (PyExceptionInstance_Check(cause)) { @@ -2408,7 +2403,7 @@ PyObject * _PyEval_GetBuiltin(PyObject *name) { PyObject *attr; - if (PyDict_GetItemRef(PyEval_GetBuiltins(), name, &attr) == 0) { + if (PyMapping_GetOptionalItem(PyEval_GetBuiltins(), name, &attr) == 0) { PyErr_SetObject(PyExc_AttributeError, name); } return attr; @@ -2561,7 +2556,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name, PyObject *fromlist, PyObject *level) { PyObject *import_func; - if (PyDict_GetItemRef(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { + if (PyMapping_GetOptionalItem(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { return NULL; } if (import_func == NULL) { @@ -2607,11 +2602,10 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name) /* Issue #17636: in case this failed because of a circular relative import, try to fallback on reading the module directly from sys.modules. */ - pkgname = PyObject_GetAttr(v, &_Py_ID(__name__)); - if (pkgname == NULL) { - goto error; + if (PyObject_GetOptionalAttr(v, &_Py_ID(__name__), &pkgname) < 0) { + return NULL; } - if (!PyUnicode_Check(pkgname)) { + if (pkgname == NULL || !PyUnicode_Check(pkgname)) { Py_CLEAR(pkgname); goto error; } @@ -2628,42 +2622,59 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name) Py_DECREF(pkgname); return x; error: - pkgpath = PyModule_GetFilenameObject(v); if (pkgname == NULL) { pkgname_or_unknown = PyUnicode_FromString(""); if (pkgname_or_unknown == NULL) { - Py_XDECREF(pkgpath); return NULL; } } else { pkgname_or_unknown = pkgname; } + pkgpath = NULL; + if (PyModule_Check(v)) { + pkgpath = PyModule_GetFilenameObject(v); + if (pkgpath == NULL) { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + Py_DECREF(pkgname_or_unknown); + return NULL; + } + // module filename missing + _PyErr_Clear(tstate); + } + } if (pkgpath == NULL || !PyUnicode_Check(pkgpath)) { - _PyErr_Clear(tstate); + Py_CLEAR(pkgpath); errmsg = PyUnicode_FromFormat( "cannot import name %R from %R (unknown location)", name, pkgname_or_unknown ); - /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ - _PyErr_SetImportErrorWithNameFrom(errmsg, pkgname, NULL, name); } else { - PyObject *spec = PyObject_GetAttr(v, &_Py_ID(__spec__)); + PyObject *spec; + int rc = PyObject_GetOptionalAttr(v, &_Py_ID(__spec__), &spec); + if (rc > 0) { + rc = _PyModuleSpec_IsInitializing(spec); + Py_DECREF(spec); + } + if (rc < 0) { + Py_DECREF(pkgname_or_unknown); + Py_DECREF(pkgpath); + return NULL; + } const char *fmt = - _PyModuleSpec_IsInitializing(spec) ? + rc ? "cannot import name %R from partially initialized module %R " "(most likely due to a circular import) (%S)" : "cannot import name %R from %R (%S)"; - Py_XDECREF(spec); errmsg = PyUnicode_FromFormat(fmt, name, pkgname_or_unknown, pkgpath); - /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ - _PyErr_SetImportErrorWithNameFrom(errmsg, pkgname, pkgpath, name); } + /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ + _PyErr_SetImportErrorWithNameFrom(errmsg, pkgname, pkgpath, name); Py_XDECREF(errmsg); - Py_XDECREF(pkgname_or_unknown); + Py_DECREF(pkgname_or_unknown); Py_XDECREF(pkgpath); return NULL; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 92c4b2fee9f863..d70abbc27606b4 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -307,10 +307,6 @@ take_gil(PyThreadState *tstate) MUTEX_LOCK(gil->mutex); - if (!_Py_atomic_load_int_relaxed(&gil->locked)) { - goto _ready; - } - int drop_requested = 0; while (_Py_atomic_load_int_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; @@ -345,7 +341,6 @@ take_gil(PyThreadState *tstate) } } -_ready: #ifdef FORCE_SWITCHING /* This mutex must be taken before modifying gil->last_holder: see drop_gil(). */ @@ -452,7 +447,7 @@ init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) interp->ceval.own_gil = 1; } -PyStatus +void _PyEval_InitGIL(PyThreadState *tstate, int own_gil) { assert(tstate->interp->ceval.gil == NULL); @@ -471,8 +466,6 @@ _PyEval_InitGIL(PyThreadState *tstate, int own_gil) // Lock the GIL and mark the current thread as attached. _PyThreadState_Attach(tstate); - - return _PyStatus_OK(); } void @@ -589,9 +582,7 @@ _PyEval_ReInitThreads(PyThreadState *tstate) take_gil(tstate); struct _pending_calls *pending = &tstate->interp->ceval.pending; - if (_PyThread_at_fork_reinit(&pending->lock) < 0) { - return _PyStatus_ERR("Can't reinitialize pending calls lock"); - } + _PyMutex_at_fork_reinit(&pending->mutex); /* Destroy all threads except the current one */ _PyThreadState_DeleteExcept(tstate); @@ -720,13 +711,10 @@ _PyEval_AddPendingCall(PyInterpreterState *interp, assert(_Py_IsMainInterpreter(interp)); pending = &_PyRuntime.ceval.pending_mainthread; } - /* Ensure that _PyEval_InitState() was called - and that _PyEval_FiniState() is not called yet. */ - assert(pending->lock != NULL); - PyThread_acquire_lock(pending->lock, WAIT_LOCK); + PyMutex_Lock(&pending->mutex); int result = _push_pending_call(pending, func, arg, flags); - PyThread_release_lock(pending->lock); + PyMutex_Unlock(&pending->mutex); /* signal main loop */ SIGNAL_PENDING_CALLS(interp); @@ -768,9 +756,9 @@ _make_pending_calls(struct _pending_calls *pending) int flags = 0; /* pop one item off the queue while holding the lock */ - PyThread_acquire_lock(pending->lock, WAIT_LOCK); + PyMutex_Lock(&pending->mutex); _pop_pending_call(pending, &func, &arg, &flags); - PyThread_release_lock(pending->lock); + PyMutex_Unlock(&pending->mutex); /* having released the lock, perform the callback */ if (func == NULL) { @@ -795,7 +783,7 @@ make_pending_calls(PyInterpreterState *interp) /* Only one thread (per interpreter) may run the pending calls at once. In the same way, we don't do recursive pending calls. */ - PyThread_acquire_lock(pending->lock, WAIT_LOCK); + PyMutex_Lock(&pending->mutex); if (pending->busy) { /* A pending call was added after another thread was already handling the pending calls (and had already "unsignaled"). @@ -807,11 +795,11 @@ make_pending_calls(PyInterpreterState *interp) care of any remaining pending calls. Until then, though, all the interpreter's threads will be tripping the eval breaker every time it's checked. */ - PyThread_release_lock(pending->lock); + PyMutex_Unlock(&pending->mutex); return 0; } pending->busy = 1; - PyThread_release_lock(pending->lock); + PyMutex_Unlock(&pending->mutex); /* unsignal before starting to call callbacks, so that any callback added in-between re-signals */ @@ -892,23 +880,9 @@ Py_MakePendingCalls(void) } void -_PyEval_InitState(PyInterpreterState *interp, PyThread_type_lock pending_lock) +_PyEval_InitState(PyInterpreterState *interp) { _gil_initialize(&interp->_gil); - - struct _pending_calls *pending = &interp->ceval.pending; - assert(pending->lock == NULL); - pending->lock = pending_lock; -} - -void -_PyEval_FiniState(struct _ceval_state *ceval) -{ - struct _pending_calls *pending = &ceval->pending; - if (pending->lock != NULL) { - PyThread_free_lock(pending->lock); - pending->lock = NULL; - } } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b0cb7c8926338c..a3606b17b71c62 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -81,7 +81,7 @@ /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef LLTRACE #define PRE_DISPATCH_GOTO() if (lltrace >= 5) { \ - lltrace_instruction(frame, stack_pointer, next_instr); } + lltrace_instruction(frame, stack_pointer, next_instr, opcode, oparg); } #else #define PRE_DISPATCH_GOTO() ((void)0) #endif @@ -258,10 +258,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { if (ADAPTIVE_COUNTER_IS_ZERO(next_instr->cache)) { \ STAT_INC((INSTNAME), deopt); \ } \ - else { \ - /* This is about to be (incorrectly) incremented: */ \ - STAT_DEC((INSTNAME), deferred); \ - } \ } while (0) #else #define UPDATE_MISS_STATS(INSTNAME) ((void)0) @@ -396,8 +392,6 @@ stack_pointer = _PyFrame_GetStackPointer(frame); #define GOTO_TIER_TWO() goto enter_tier_two; -#define GOTO_TIER_ONE() goto exit_trace; - #define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND() (next_uop[-1].operand) diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 98717ecc875b8b..93b8385a5b4097 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -289,6 +289,40 @@ sys_intern(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(sys__is_interned__doc__, +"_is_interned($module, string, /)\n" +"--\n" +"\n" +"Return True if the given string is \"interned\"."); + +#define SYS__IS_INTERNED_METHODDEF \ + {"_is_interned", (PyCFunction)sys__is_interned, METH_O, sys__is_interned__doc__}, + +static int +sys__is_interned_impl(PyObject *module, PyObject *string); + +static PyObject * +sys__is_interned(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *string; + int _return_value; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_is_interned", "argument", "str", arg); + goto exit; + } + string = arg; + _return_value = sys__is_interned_impl(module, string); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(sys__settraceallthreads__doc__, "_settraceallthreads($module, arg, /)\n" "--\n" @@ -1452,4 +1486,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=f36d45c829250775 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3dc3b2cb0ce38ebb input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index b79bf555f2f22a..d8fe7b22063a80 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -146,15 +146,14 @@ PyObject *_PyCodec_Lookup(const char *encoding) PyUnicode_InternInPlace(&v); /* First, try to lookup the name in the registry dictionary */ - PyObject *result = PyDict_GetItemWithError(interp->codec_search_cache, v); + PyObject *result; + if (PyDict_GetItemRef(interp->codec_search_cache, v, &result) < 0) { + goto onError; + } if (result != NULL) { - Py_INCREF(result); Py_DECREF(v); return result; } - else if (PyErr_Occurred()) { - goto onError; - } /* Next, scan the search functions in order of registration */ const Py_ssize_t len = PyList_Size(interp->codec_search_path); @@ -932,8 +931,6 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) return Py_BuildValue("(Nn)", res, end); } -static _PyUnicode_Name_CAPI *ucnhash_capi = NULL; - PyObject *PyCodec_NameReplaceErrors(PyObject *exc) { if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { @@ -954,13 +951,9 @@ PyObject *PyCodec_NameReplaceErrors(PyObject *exc) return NULL; if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; - if (!ucnhash_capi) { - /* load the unicode data module */ - ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( - PyUnicodeData_CAPSULE_NAME, 1); - if (!ucnhash_capi) { - return NULL; - } + _PyUnicode_Name_CAPI *ucnhash_capi = _PyUnicode_GetNameCAPI(); + if (ucnhash_capi == NULL) { + return NULL; } for (i = start, ressize = 0; i < end; ++i) { /* object is guaranteed to be "ready" */ diff --git a/Python/compile.c b/Python/compile.c index 8b9e2f02048f11..65ac05ad58d4dd 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -796,35 +796,12 @@ stack_effect(int opcode, int oparg, int jump) // Specialized instructions are not supported. return PY_INVALID_STACK_EFFECT; } - int popped, pushed; - if (jump > 0) { - popped = _PyOpcode_num_popped(opcode, oparg, true); - pushed = _PyOpcode_num_pushed(opcode, oparg, true); - } - else { - popped = _PyOpcode_num_popped(opcode, oparg, false); - pushed = _PyOpcode_num_pushed(opcode, oparg, false); - } + int popped = _PyOpcode_num_popped(opcode, oparg); + int pushed = _PyOpcode_num_pushed(opcode, oparg); if (popped < 0 || pushed < 0) { return PY_INVALID_STACK_EFFECT; } - if (jump >= 0) { - return pushed - popped; - } - if (jump < 0) { - // Compute max(pushed - popped, alt_pushed - alt_popped) - int alt_popped = _PyOpcode_num_popped(opcode, oparg, true); - int alt_pushed = _PyOpcode_num_pushed(opcode, oparg, true); - if (alt_popped < 0 || alt_pushed < 0) { - return PY_INVALID_STACK_EFFECT; - } - int diff = pushed - popped; - int alt_diff = alt_pushed - alt_popped; - if (alt_diff > diff) { - return alt_diff; - } - return diff; - } + return pushed - popped; } // Pseudo ops @@ -1125,7 +1102,7 @@ compiler_addop_name(struct compiler_unit *u, location loc, arg <<= 1; } if (opcode == LOAD_METHOD) { - assert(SAME_OPCODE_METADATA(LOAD_METHOD, LOAD_ATTR)); + assert(is_pseudo_target(LOAD_METHOD, LOAD_ATTR)); opcode = LOAD_ATTR; arg <<= 1; arg |= 1; @@ -1135,18 +1112,18 @@ compiler_addop_name(struct compiler_unit *u, location loc, arg |= 2; } if (opcode == LOAD_SUPER_METHOD) { - assert(SAME_OPCODE_METADATA(LOAD_SUPER_METHOD, LOAD_SUPER_ATTR)); + assert(is_pseudo_target(LOAD_SUPER_METHOD, LOAD_SUPER_ATTR)); opcode = LOAD_SUPER_ATTR; arg <<= 2; arg |= 3; } if (opcode == LOAD_ZERO_SUPER_ATTR) { - assert(SAME_OPCODE_METADATA(LOAD_ZERO_SUPER_ATTR, LOAD_SUPER_ATTR)); + assert(is_pseudo_target(LOAD_ZERO_SUPER_ATTR, LOAD_SUPER_ATTR)); opcode = LOAD_SUPER_ATTR; arg <<= 2; } if (opcode == LOAD_ZERO_SUPER_METHOD) { - assert(SAME_OPCODE_METADATA(LOAD_ZERO_SUPER_METHOD, LOAD_SUPER_ATTR)); + assert(is_pseudo_target(LOAD_ZERO_SUPER_METHOD, LOAD_SUPER_ATTR)); opcode = LOAD_SUPER_ATTR; arg <<= 2; arg |= 1; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 21b96ef05ed799..c6ed7daeb1074a 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -12,6 +12,53 @@ #include "pycore_weakref.h" // _PyWeakref_GET_REF() +/**************/ +/* exceptions */ +/**************/ + +/* InterpreterError extends Exception */ + +static PyTypeObject _PyExc_InterpreterError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "InterpreterError", + .tp_doc = PyDoc_STR("An interpreter was not found."), + //.tp_base = (PyTypeObject *)PyExc_BaseException, +}; +PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; + +/* InterpreterNotFoundError extends InterpreterError */ + +static PyTypeObject _PyExc_InterpreterNotFoundError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "InterpreterNotFoundError", + .tp_doc = PyDoc_STR("An interpreter was not found."), + .tp_base = &_PyExc_InterpreterError, +}; +PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; + +/* lifecycle */ + +static int +init_exceptions(PyInterpreterState *interp) +{ + _PyExc_InterpreterError.tp_base = (PyTypeObject *)PyExc_BaseException; + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { + return -1; + } + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { + return -1; + } + return 0; +} + +static void +fini_exceptions(PyInterpreterState *interp) +{ + _PyStaticType_Dealloc(interp, &_PyExc_InterpreterNotFoundError); + _PyStaticType_Dealloc(interp, &_PyExc_InterpreterError); +} + + /***************************/ /* cross-interpreter calls */ /***************************/ @@ -456,16 +503,17 @@ _xidregistry_clear(struct _xidregistry *xidregistry) static void _xidregistry_lock(struct _xidregistry *registry) { - if (registry->mutex != NULL) { - PyThread_acquire_lock(registry->mutex, WAIT_LOCK); + if (registry->global) { + PyMutex_Lock(®istry->mutex); } + // else: Within an interpreter we rely on the GIL instead of a separate lock. } static void _xidregistry_unlock(struct _xidregistry *registry) { - if (registry->mutex != NULL) { - PyThread_release_lock(registry->mutex); + if (registry->global) { + PyMutex_Unlock(®istry->mutex); } } @@ -874,19 +922,10 @@ _xidregistry_init(struct _xidregistry *registry) registry->initialized = 1; if (registry->global) { - // We manage the mutex lifecycle in pystate.c. - assert(registry->mutex != NULL); - // Registering the builtins is cheap so we don't bother doing it lazily. assert(registry->head == NULL); _register_builtins_for_crossinterpreter_data(registry); } - else { - // Within an interpreter we rely on the GIL instead of a separate lock. - assert(registry->mutex == NULL); - - // There's nothing else to initialize. - } } static void @@ -898,17 +937,6 @@ _xidregistry_fini(struct _xidregistry *registry) registry->initialized = 0; _xidregistry_clear(registry); - - if (registry->global) { - // We manage the mutex lifecycle in pystate.c. - assert(registry->mutex != NULL); - } - else { - // There's nothing else to finalize. - - // Within an interpreter we rely on the GIL instead of a separate lock. - assert(registry->mutex == NULL); - } } @@ -917,22 +945,108 @@ _xidregistry_fini(struct _xidregistry *registry) /*************************/ static const char * -_copy_string_obj_raw(PyObject *strobj) +_copy_string_obj_raw(PyObject *strobj, Py_ssize_t *p_size) { - const char *str = PyUnicode_AsUTF8(strobj); + Py_ssize_t size = -1; + const char *str = PyUnicode_AsUTF8AndSize(strobj, &size); if (str == NULL) { return NULL; } - char *copied = PyMem_RawMalloc(strlen(str)+1); + char *copied = PyMem_RawMalloc(size+1); if (copied == NULL) { PyErr_NoMemory(); return NULL; } strcpy(copied, str); + if (p_size != NULL) { + *p_size = size; + } return copied; } + +static int +_convert_exc_to_TracebackException(PyObject *exc, PyObject **p_tbexc) +{ + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *create = NULL; + + // This is inspired by _PyErr_Display(). + PyObject *tbmod = PyImport_ImportModule("traceback"); + if (tbmod == NULL) { + return -1; + } + PyObject *tbexc_type = PyObject_GetAttrString(tbmod, "TracebackException"); + Py_DECREF(tbmod); + if (tbexc_type == NULL) { + return -1; + } + create = PyObject_GetAttrString(tbexc_type, "from_exception"); + Py_DECREF(tbexc_type); + if (create == NULL) { + return -1; + } + + args = PyTuple_Pack(1, exc); + if (args == NULL) { + goto error; + } + + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto error; + } + if (PyDict_SetItemString(kwargs, "save_exc_type", Py_False) < 0) { + goto error; + } + if (PyDict_SetItemString(kwargs, "lookup_lines", Py_False) < 0) { + goto error; + } + + PyObject *tbexc = PyObject_Call(create, args, kwargs); + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(create); + if (tbexc == NULL) { + goto error; + } + + *p_tbexc = tbexc; + return 0; + +error: + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(create); + return -1; +} + + +static const char * +_format_TracebackException(PyObject *tbexc) +{ + PyObject *lines = PyObject_CallMethod(tbexc, "format", NULL); + if (lines == NULL) { + return NULL; + } + PyObject *formatted_obj = PyUnicode_Join(&_Py_STR(empty), lines); + Py_DECREF(lines); + if (formatted_obj == NULL) { + return NULL; + } + + Py_ssize_t size = -1; + const char *formatted = _copy_string_obj_raw(formatted_obj, &size); + Py_DECREF(formatted_obj); + // We remove trailing the newline added by TracebackException.format(). + assert(formatted[size-1] == '\n'); + ((char *)formatted)[size-1] = '\0'; + return formatted; +} + + static int _release_xid_data(_PyCrossInterpreterData *data, int rawfree) { @@ -979,7 +1093,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) if (strobj == NULL) { return -1; } - info->name = _copy_string_obj_raw(strobj); + info->name = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); if (info->name == NULL) { return -1; @@ -990,7 +1104,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) if (strobj == NULL) { return -1; } - info->qualname = _copy_string_obj_raw(strobj); + info->qualname = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); if (info->name == NULL) { return -1; @@ -1001,7 +1115,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) if (strobj == NULL) { return -1; } - info->module = _copy_string_obj_raw(strobj); + info->module = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); if (info->name == NULL) { return -1; @@ -1066,6 +1180,9 @@ _PyXI_excinfo_Clear(_PyXI_excinfo *info) if (info->msg != NULL) { PyMem_RawFree((void *)info->msg); } + if (info->errdisplay != NULL) { + PyMem_RawFree((void *)info->errdisplay); + } *info = (_PyXI_excinfo){{NULL}}; } @@ -1123,13 +1240,32 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) failure = "error while formatting exception"; goto error; } - info->msg = _copy_string_obj_raw(msgobj); + info->msg = _copy_string_obj_raw(msgobj, NULL); Py_DECREF(msgobj); if (info->msg == NULL) { failure = "error while copying exception message"; goto error; } + // Pickle a traceback.TracebackException. + PyObject *tbexc = NULL; + if (_convert_exc_to_TracebackException(exc, &tbexc) < 0) { +#ifdef Py_DEBUG + PyErr_FormatUnraisable("Exception ignored while creating TracebackException"); +#endif + PyErr_Clear(); + } + else { + info->errdisplay = _format_TracebackException(tbexc); + Py_DECREF(tbexc); + if (info->errdisplay == NULL) { +#ifdef Py_DEBUG + PyErr_FormatUnraisable("Exception ignored while formating TracebackException"); +#endif + PyErr_Clear(); + } + } + return NULL; error: @@ -1141,9 +1277,29 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) static void _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) { + PyObject *tbexc = NULL; + if (info->errdisplay != NULL) { + tbexc = PyUnicode_FromString(info->errdisplay); + if (tbexc == NULL) { + PyErr_Clear(); + } + } + PyObject *formatted = _PyXI_excinfo_format(info); PyErr_SetObject(exctype, formatted); Py_DECREF(formatted); + + if (tbexc != NULL) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyObject_SetAttrString(exc, "_errdisplay", tbexc) < 0) { +#ifdef Py_DEBUG + PyErr_FormatUnraisable("Exception ignored when setting _errdisplay"); +#endif + PyErr_Clear(); + } + Py_DECREF(tbexc); + PyErr_SetRaisedException(exc); + } } static PyObject * @@ -1249,6 +1405,20 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info) goto error; } + if (info->errdisplay != NULL) { + PyObject *tbexc = PyUnicode_FromString(info->errdisplay); + if (tbexc == NULL) { + PyErr_Clear(); + } + else { + res = PyObject_SetAttrString(ns, "errdisplay", tbexc); + Py_DECREF(tbexc); + if (res < 0) { + goto error; + } + } + } + return ns; error: @@ -1413,7 +1583,7 @@ _sharednsitem_is_initialized(_PyXI_namespace_item *item) static int _sharednsitem_init(_PyXI_namespace_item *item, PyObject *key) { - item->name = _copy_string_obj_raw(key); + item->name = _copy_string_obj_raw(key, NULL); if (item->name == NULL) { assert(!_sharednsitem_is_initialized(item)); return -1; @@ -1955,6 +2125,7 @@ _capture_current_exception(_PyXI_session *session) } else { failure = _PyXI_InitError(err, excval, _PyXI_ERR_UNCAUGHT_EXCEPTION); + Py_DECREF(excval); if (failure == NULL && override != NULL) { err->code = errcode; } @@ -1969,18 +2140,6 @@ _capture_current_exception(_PyXI_session *session) err = NULL; } - // a temporary hack (famous last words) - if (excval != NULL) { - // XXX Store the traceback info (or rendered traceback) on - // _PyXI_excinfo, attach it to the exception when applied, - // and teach PyErr_Display() to print it. -#ifdef Py_DEBUG - // XXX Drop this once _Py_excinfo picks up the slack. - PyErr_Display(NULL, excval, NULL); -#endif - Py_DECREF(excval); - } - // Finished! assert(!PyErr_Occurred()); session->error = err; @@ -2118,3 +2277,18 @@ _PyXI_Fini(PyInterpreterState *interp) _xidregistry_fini(_get_global_xidregistry(interp->runtime)); } } + +PyStatus +_PyXI_InitTypes(PyInterpreterState *interp) +{ + if (init_exceptions(interp) < 0) { + return _PyStatus_ERR("failed to initialize an exception type"); + } + return _PyStatus_OK(); +} + +void +_PyXI_FiniTypes(PyInterpreterState *interp) +{ + fini_exceptions(interp); +} diff --git a/Python/dtoa.c b/Python/dtoa.c index 5dfc0e179cbc34..6e3162f80bdae1 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -309,7 +309,7 @@ BCinfo { // struct Bigint is defined in pycore_dtoa.h. typedef struct Bigint Bigint; -#ifndef Py_USING_MEMORY_DEBUGGER +#if !defined(Py_GIL_DISABLED) && !defined(Py_USING_MEMORY_DEBUGGER) /* Memory management: memory is allocated from, and returned to, Kmax+1 pools of memory, where pool k (0 <= k <= Kmax) is for Bigints b with b->maxwds == @@ -428,7 +428,7 @@ Bfree(Bigint *v) } } -#endif /* Py_USING_MEMORY_DEBUGGER */ +#endif /* !defined(Py_GIL_DISABLED) && !defined(Py_USING_MEMORY_DEBUGGER) */ #define Bcopy(x,y) memcpy((char *)&x->sign, (char *)&y->sign, \ y->wds*sizeof(Long) + 2*sizeof(int)) @@ -673,10 +673,17 @@ mult(Bigint *a, Bigint *b) static Bigint * pow5mult(Bigint *b, int k) { - Bigint *b1, *p5, *p51; + Bigint *b1, *p5, **p5s; int i; static const int p05[3] = { 5, 25, 125 }; + // For double-to-string conversion, the maximum value of k is limited by + // DBL_MAX_10_EXP (308), the maximum decimal base-10 exponent for binary64. + // For string-to-double conversion, the extreme case is constrained by our + // hardcoded exponent limit before we underflow of -512, adjusted by + // STRTOD_DIGLIM-DBL_DIG-1, giving a maximum of k=535. + assert(0 <= k && k < 1024); + if ((i = k & 3)) { b = multadd(b, p05[i-1], 0); if (b == NULL) @@ -686,18 +693,11 @@ pow5mult(Bigint *b, int k) if (!(k >>= 2)) return b; PyInterpreterState *interp = _PyInterpreterState_GET(); - p5 = interp->dtoa.p5s; - if (!p5) { - /* first time */ - p5 = i2b(625); - if (p5 == NULL) { - Bfree(b); - return NULL; - } - interp->dtoa.p5s = p5; - p5->next = 0; - } + p5s = interp->dtoa.p5s; for(;;) { + assert(p5s != interp->dtoa.p5s + Bigint_Pow5size); + p5 = *p5s; + p5s++; if (k & 1) { b1 = mult(b, p5); Bfree(b); @@ -707,17 +707,6 @@ pow5mult(Bigint *b, int k) } if (!(k >>= 1)) break; - p51 = p5->next; - if (!p51) { - p51 = mult(p5,p5); - if (p51 == NULL) { - Bfree(b); - return NULL; - } - p51->next = 0; - p5->next = p51; - } - p5 = p51; } return b; } @@ -2811,3 +2800,42 @@ _Py_dg_dtoa(double dd, int mode, int ndigits, } #endif // _PY_SHORT_FLOAT_REPR == 1 + +PyStatus +_PyDtoa_Init(PyInterpreterState *interp) +{ +#if _PY_SHORT_FLOAT_REPR == 1 && !defined(Py_USING_MEMORY_DEBUGGER) + Bigint **p5s = interp->dtoa.p5s; + + // 5**4 = 625 + Bigint *p5 = i2b(625); + if (p5 == NULL) { + return PyStatus_NoMemory(); + } + p5s[0] = p5; + + // compute 5**8, 5**16, 5**32, ..., 5**512 + for (Py_ssize_t i = 1; i < Bigint_Pow5size; i++) { + p5 = mult(p5, p5); + if (p5 == NULL) { + return PyStatus_NoMemory(); + } + p5s[i] = p5; + } + +#endif + return PyStatus_Ok(); +} + +void +_PyDtoa_Fini(PyInterpreterState *interp) +{ +#if _PY_SHORT_FLOAT_REPR == 1 && !defined(Py_USING_MEMORY_DEBUGGER) + Bigint **p5s = interp->dtoa.p5s; + for (Py_ssize_t i = 0; i < Bigint_Pow5size; i++) { + Bigint *p5 = p5s[i]; + p5s[i] = NULL; + Bfree(p5); + } +#endif +} diff --git a/Python/errors.c b/Python/errors.c index ed5eec5c261970..e5f176a5dd208e 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -121,11 +121,11 @@ _PyErr_GetTopmostException(PyThreadState *tstate) _PyErr_StackItem *exc_info = tstate->exc_info; assert(exc_info); - while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && - exc_info->previous_item != NULL) + while (exc_info->exc_value == NULL && exc_info->previous_item != NULL) { exc_info = exc_info->previous_item; } + assert(!Py_IsNone(exc_info->exc_value)); return exc_info; } @@ -592,7 +592,7 @@ PyErr_GetHandledException(void) void _PyErr_SetHandledException(PyThreadState *tstate, PyObject *exc) { - Py_XSETREF(tstate->exc_info->exc_value, Py_XNewRef(exc)); + Py_XSETREF(tstate->exc_info->exc_value, Py_XNewRef(exc == Py_None ? NULL : exc)); } void diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 547be6f13237dd..14fb3a05a9f674 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1,4 +1,4 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/tier2_generator.py // from: // Python/bytecodes.c // Do not edit! @@ -8,102 +8,104 @@ #endif #define TIER_TWO 2 - case NOP: { + case _NOP: { break; } - case RESUME_CHECK: { -#if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + case _RESUME_CHECK: { + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) goto deoptimize; _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; -#endif + #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + if (eval_breaker != version) goto deoptimize; break; } - case LOAD_FAST_CHECK: { - oparg = CURRENT_OPARG(); + /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + + case _LOAD_FAST_CHECK: { PyObject *value; + oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error_tier_two; Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case LOAD_FAST: { - oparg = CURRENT_OPARG(); + case _LOAD_FAST: { PyObject *value; + oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case LOAD_FAST_AND_CLEAR: { - oparg = CURRENT_OPARG(); + case _LOAD_FAST_AND_CLEAR: { PyObject *value; + oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case LOAD_CONST: { - oparg = CURRENT_OPARG(); + case _LOAD_CONST: { PyObject *value; + oparg = CURRENT_OPARG(); value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case STORE_FAST: { - oparg = CURRENT_OPARG(); + case _STORE_FAST: { PyObject *value; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; SETLOCAL(oparg, value); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case POP_TOP: { + case _POP_TOP: { PyObject *value; value = stack_pointer[-1]; Py_DECREF(value); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case PUSH_NULL: { + case _PUSH_NULL: { PyObject *res; res = NULL; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case END_SEND: { + case _END_SEND: { PyObject *value; PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; break; } - case UNARY_NEGATIVE: { + case _UNARY_NEGATIVE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -114,7 +116,7 @@ break; } - case UNARY_NOT: { + case _UNARY_NOT: { PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -136,19 +138,19 @@ break; } - case TO_BOOL_BOOL: { + case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + if (!PyBool_Check(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); break; } - case TO_BOOL_INT: { + case _TO_BOOL_INT: { PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + if (!PyLong_CheckExact(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -162,11 +164,11 @@ break; } - case TO_BOOL_LIST: { + case _TO_BOOL_LIST: { PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + if (!PyList_CheckExact(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -174,23 +176,23 @@ break; } - case TO_BOOL_NONE: { + case _TO_BOOL_NONE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + if (!Py_IsNone(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; break; } - case TO_BOOL_STR: { + case _TO_BOOL_STR: { PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + if (!PyUnicode_CheckExact(value)) goto deoptimize; STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -205,14 +207,14 @@ break; } - case TO_BOOL_ALWAYS_TRUE: { + case _TO_BOOL_ALWAYS_TRUE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; uint32_t version = (uint32_t)CURRENT_OPERAND(); // This one is a bit weird, because we expect *some* failures: assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + if (Py_TYPE(value)->tp_version_tag != version) goto deoptimize; STAT_INC(TO_BOOL, hit); Py_DECREF(value); res = Py_True; @@ -220,7 +222,7 @@ break; } - case UNARY_INVERT: { + case _UNARY_INVERT: { PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -236,8 +238,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT); - DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT); + if (!PyLong_CheckExact(left)) goto deoptimize; + if (!PyLong_CheckExact(right)) goto deoptimize; break; } @@ -252,8 +254,8 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -268,8 +270,8 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -284,8 +286,8 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -294,8 +296,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT); - DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT); + if (!PyFloat_CheckExact(left)) goto deoptimize; + if (!PyFloat_CheckExact(right)) goto deoptimize; break; } @@ -307,11 +309,11 @@ left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -323,11 +325,11 @@ left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -339,11 +341,11 @@ left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -352,8 +354,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE); - DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE); + if (!PyUnicode_CheckExact(left)) goto deoptimize; + if (!PyUnicode_CheckExact(right)) goto deoptimize; break; } @@ -368,8 +370,8 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } @@ -383,12 +385,12 @@ Py_DECREF(container); Py_DECREF(sub); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case BINARY_SLICE: { + case _BINARY_SLICE: { PyObject *stop; PyObject *start; PyObject *container; @@ -408,12 +410,12 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error_tier_two; - STACK_SHRINK(2); - stack_pointer[-1] = res; + stack_pointer[-3] = res; + stack_pointer += -2; break; } - case STORE_SLICE: { + case _STORE_SLICE: { PyObject *stop; PyObject *start; PyObject *container; @@ -434,88 +436,86 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error_tier_two; - STACK_SHRINK(4); + stack_pointer += -4; break; } - case BINARY_SUBSCR_LIST_INT: { + case _BINARY_SUBSCR_LIST_INT: { PyObject *sub; PyObject *list; PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - + if (!PyLong_CheckExact(sub)) goto deoptimize; + if (!PyList_CheckExact(list)) goto deoptimize; // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + if (index >= PyList_GET_SIZE(list)) goto deoptimize; STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case BINARY_SUBSCR_STR_INT: { + case _BINARY_SUBSCR_STR_INT: { PyObject *sub; PyObject *str; PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) goto deoptimize; + if (!PyUnicode_CheckExact(str)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + if (PyUnicode_GET_LENGTH(str) <= index) goto deoptimize; // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) goto deoptimize; STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(str); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case BINARY_SUBSCR_TUPLE_INT: { + case _BINARY_SUBSCR_TUPLE_INT: { PyObject *sub; PyObject *tuple; PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - + if (!PyLong_CheckExact(sub)) goto deoptimize; + if (!PyTuple_CheckExact(tuple)) goto deoptimize; // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + if (index >= PyTuple_GET_SIZE(tuple)) goto deoptimize; STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case BINARY_SUBSCR_DICT: { + case _BINARY_SUBSCR_DICT: { PyObject *sub; PyObject *dict; PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + if (!PyDict_CheckExact(dict)) goto deoptimize; STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); if (rc == 0) { @@ -524,32 +524,35 @@ Py_DECREF(dict); Py_DECREF(sub); if (rc <= 0) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case LIST_APPEND: { - oparg = CURRENT_OPARG(); + /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */ + + case _LIST_APPEND: { PyObject *v; PyObject *list; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case SET_ADD: { - oparg = CURRENT_OPARG(); + case _SET_ADD: { PyObject *v; PyObject *set; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } @@ -566,54 +569,52 @@ Py_DECREF(container); Py_DECREF(sub); if (err) goto pop_3_error_tier_two; - STACK_SHRINK(3); + stack_pointer += -3; break; } - case STORE_SUBSCR_LIST_INT: { + case _STORE_SUBSCR_LIST_INT: { PyObject *sub; PyObject *list; PyObject *value; sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - + if (!PyLong_CheckExact(sub)) goto deoptimize; + if (!PyList_CheckExact(list)) goto deoptimize; // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + if (index >= PyList_GET_SIZE(list)) goto deoptimize; STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(3); + stack_pointer += -3; break; } - case STORE_SUBSCR_DICT: { + case _STORE_SUBSCR_DICT: { PyObject *sub; PyObject *dict; PyObject *value; sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + if (!PyDict_CheckExact(dict)) goto deoptimize; STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error_tier_two; - STACK_SHRINK(3); + stack_pointer += -3; break; } - case DELETE_SUBSCR: { + case _DELETE_SUBSCR: { PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -623,14 +624,14 @@ Py_DECREF(container); Py_DECREF(sub); if (err) goto pop_2_error_tier_two; - STACK_SHRINK(2); + stack_pointer += -2; break; } - case CALL_INTRINSIC_1: { - oparg = CURRENT_OPARG(); + case _CALL_INTRINSIC_1: { PyObject *value; PyObject *res; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); @@ -640,11 +641,11 @@ break; } - case CALL_INTRINSIC_2: { - oparg = CURRENT_OPARG(); + case _CALL_INTRINSIC_2: { PyObject *value1; PyObject *value2; PyObject *res; + oparg = CURRENT_OPARG(); value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); @@ -652,20 +653,20 @@ Py_DECREF(value2); Py_DECREF(value1); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } case _POP_FRAME: { PyObject *retval; retval = stack_pointer[-1]; - STACK_SHRINK(1); - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -674,26 +675,28 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif + #endif break; } - case GET_AITER: { + /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ + + case _GET_AITER: { PyObject *obj; PyObject *iter; obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " @@ -702,14 +705,11 @@ Py_DECREF(obj); if (true) goto pop_1_error_tier_two; } - iter = (*getter)(obj); Py_DECREF(obj); if (iter == NULL) goto pop_1_error_tier_two; - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", @@ -721,14 +721,13 @@ break; } - case GET_ANEXT: { + case _GET_ANEXT: { PyObject *aiter; PyObject *awaitable; aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { @@ -738,7 +737,6 @@ if (type->tp_as_async != NULL){ getter = type->tp_as_async->am_anext; } - if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { @@ -752,7 +750,6 @@ type->tp_name); GOTO_ERROR(error); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { _PyErr_FormatFromCause( @@ -760,31 +757,27 @@ "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); GOTO_ERROR(error); } else { Py_DECREF(next_iter); } } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + stack_pointer[0] = awaitable; + stack_pointer += 1; break; } - case GET_AWAITABLE: { - oparg = CURRENT_OPARG(); + case _GET_AWAITABLE: { PyObject *iterable; PyObject *iter; + oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { @@ -798,30 +791,35 @@ /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error_tier_two; stack_pointer[-1] = iter; break; } - case POP_EXCEPT: { + /* _SEND is not a viable micro-op for tier 2 */ + + /* _SEND_GEN is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + + case _POP_EXCEPT: { PyObject *exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - STACK_SHRINK(1); + Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + stack_pointer += -1; break; } - case LOAD_ASSERTION_ERROR: { + case _LOAD_ASSERTION_ERROR: { PyObject *value; value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case LOAD_BUILD_CLASS: { + case _LOAD_BUILD_CLASS: { PyObject *bc; if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error_tier_two; if (bc == NULL) { @@ -829,14 +827,14 @@ "__build_class__ not found"); if (true) goto error_tier_two; } - STACK_GROW(1); - stack_pointer[-1] = bc; + stack_pointer[0] = bc; + stack_pointer += 1; break; } - case STORE_NAME: { - oparg = CURRENT_OPARG(); + case _STORE_NAME: { PyObject *v; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -848,16 +846,16 @@ if (true) goto pop_1_error_tier_two; } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, v); else - err = PyObject_SetItem(ns, name, v); + err = PyObject_SetItem(ns, name, v); Py_DECREF(v); if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case DELETE_NAME: { + case _DELETE_NAME: { oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -871,99 +869,95 @@ // Can't use ERROR_IF here. if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + NAME_ERROR_MSG, + name); GOTO_ERROR(error); } break; } case _UNPACK_SEQUENCE: { - oparg = CURRENT_OPARG(); PyObject *seq; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); if (res == 0) goto pop_1_error_tier_two; - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; break; } - case UNPACK_SEQUENCE_TWO_TUPLE: { - oparg = CURRENT_OPARG(); + case _UNPACK_SEQUENCE_TWO_TUPLE: { PyObject *seq; PyObject **values; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + values = &stack_pointer[-1]; + if (!PyTuple_CheckExact(seq)) goto deoptimize; + if (PyTuple_GET_SIZE(seq) != 2) goto deoptimize; assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; break; } - case UNPACK_SEQUENCE_TUPLE: { - oparg = CURRENT_OPARG(); + case _UNPACK_SEQUENCE_TUPLE: { PyObject *seq; PyObject **values; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + values = &stack_pointer[-1]; + if (!PyTuple_CheckExact(seq)) goto deoptimize; + if (PyTuple_GET_SIZE(seq) != oparg) goto deoptimize; STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; break; } - case UNPACK_SEQUENCE_LIST: { - oparg = CURRENT_OPARG(); + case _UNPACK_SEQUENCE_LIST: { PyObject *seq; PyObject **values; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + values = &stack_pointer[-1]; + if (!PyList_CheckExact(seq)) goto deoptimize; + if (PyList_GET_SIZE(seq) != oparg) goto deoptimize; STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; break; } - case UNPACK_EX: { - oparg = CURRENT_OPARG(); + case _UNPACK_EX: { PyObject *seq; + oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); if (res == 0) goto pop_1_error_tier_two; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; } case _STORE_ATTR: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *v; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -971,35 +965,35 @@ Py_DECREF(v); Py_DECREF(owner); if (err) goto pop_2_error_tier_two; - STACK_SHRINK(2); + stack_pointer += -2; break; } - case DELETE_ATTR: { - oparg = CURRENT_OPARG(); + case _DELETE_ATTR: { PyObject *owner; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case STORE_GLOBAL: { - oparg = CURRENT_OPARG(); + case _STORE_GLOBAL: { PyObject *v; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case DELETE_GLOBAL: { + case _DELETE_GLOBAL: { oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; @@ -1008,14 +1002,14 @@ if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } GOTO_ERROR(error); } break; } - case LOAD_LOCALS: { + case _LOAD_LOCALS: { PyObject *locals; locals = LOCALS(); if (locals == NULL) { @@ -1024,15 +1018,15 @@ if (true) goto error_tier_two; } Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; + stack_pointer[0] = locals; + stack_pointer += 1; break; } - case LOAD_FROM_DICT_OR_GLOBALS: { - oparg = CURRENT_OPARG(); + case _LOAD_FROM_DICT_OR_GLOBALS: { PyObject *mod_or_class_dict; PyObject *v; + oparg = CURRENT_OPARG(); mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { @@ -1048,8 +1042,8 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } @@ -1059,9 +1053,9 @@ break; } - case LOAD_NAME: { - oparg = CURRENT_OPARG(); + case _LOAD_NAME: { PyObject *v; + oparg = CURRENT_OPARG(); PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1082,34 +1076,34 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } } - STACK_GROW(1); - stack_pointer[-1] = v; + stack_pointer[0] = v; + stack_pointer += 1; break; } case _LOAD_GLOBAL: { - oparg = CURRENT_OPARG(); PyObject *res; PyObject *null = NULL; + oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); + (PyDictObject *)BUILTINS(), + name); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } if (true) goto error_tier_two; } @@ -1124,25 +1118,24 @@ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error_tier_two; if (res == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); if (true) goto error_tier_two; } } } null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); break; } case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); + if (!PyDict_CheckExact(dict)) goto deoptimize; + if (dict->ma_keys->dk_version != version) goto deoptimize; assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1150,51 +1143,49 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); + if (!PyDict_CheckExact(dict)) goto deoptimize; + if (dict->ma_keys->dk_version != version) goto deoptimize; assert(DK_IS_UNICODE(dict->ma_keys)); break; } case _LOAD_GLOBAL_MODULE: { - oparg = CURRENT_OPARG(); PyObject *res; PyObject *null = NULL; + oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE); + if (res == NULL) goto deoptimize; Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); break; } case _LOAD_GLOBAL_BUILTINS: { - oparg = CURRENT_OPARG(); PyObject *res; PyObject *null = NULL; + oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS); + if (res == NULL) goto deoptimize; Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); break; } - case DELETE_FAST: { + case _DELETE_FAST: { oparg = CURRENT_OPARG(); PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error_tier_two; @@ -1202,7 +1193,7 @@ break; } - case MAKE_CELL: { + case _MAKE_CELL: { oparg = CURRENT_OPARG(); // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). @@ -1215,7 +1206,7 @@ break; } - case DELETE_DEREF: { + case _DELETE_DEREF: { oparg = CURRENT_OPARG(); PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); @@ -1230,10 +1221,10 @@ break; } - case LOAD_FROM_DICT_OR_DEREF: { - oparg = CURRENT_OPARG(); + case _LOAD_FROM_DICT_OR_DEREF: { PyObject *class_dict; PyObject *value; + oparg = CURRENT_OPARG(); class_dict = stack_pointer[-1]; PyObject *name; assert(class_dict); @@ -1256,9 +1247,9 @@ break; } - case LOAD_DEREF: { - oparg = CURRENT_OPARG(); + case _LOAD_DEREF: { PyObject *value; + oparg = CURRENT_OPARG(); PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1266,24 +1257,24 @@ if (true) goto error_tier_two; } Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; break; } - case STORE_DEREF: { - oparg = CURRENT_OPARG(); + case _STORE_DEREF: { PyObject *v; + oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case COPY_FREE_VARS: { + case _COPY_FREE_VARS: { oparg = CURRENT_OPARG(); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); @@ -1298,131 +1289,126 @@ break; } - case BUILD_STRING: { - oparg = CURRENT_OPARG(); + case _BUILD_STRING: { PyObject **pieces; PyObject *str; - pieces = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; + if (str == NULL) { stack_pointer += -oparg; goto error_tier_two; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; break; } - case BUILD_TUPLE: { - oparg = CURRENT_OPARG(); + case _BUILD_TUPLE: { PyObject **values; PyObject *tup; - values = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + if (tup == NULL) { stack_pointer += -oparg; goto error_tier_two; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; break; } - case BUILD_LIST: { - oparg = CURRENT_OPARG(); + case _BUILD_LIST: { PyObject **values; PyObject *list; - values = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; + if (list == NULL) { stack_pointer += -oparg; goto error_tier_two; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; break; } - case LIST_EXTEND: { - oparg = CURRENT_OPARG(); + case _LIST_EXTEND: { PyObject *iterable; PyObject *list; + oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); if (true) goto pop_1_error_tier_two; } assert(Py_IsNone(none_val)); Py_DECREF(iterable); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case SET_UPDATE: { - oparg = CURRENT_OPARG(); + case _SET_UPDATE: { PyObject *iterable; PyObject *set; + oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error_tier_two; - STACK_SHRINK(1); + stack_pointer += -1; break; } - case BUILD_SET: { - oparg = CURRENT_OPARG(); + case _BUILD_SET: { PyObject **values; PyObject *set; - values = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + GOTO_ERROR(error); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; if (err == 0) - err = PySet_Add(set, item); + err = PySet_Add(set, item); Py_DECREF(item); } if (err != 0) { Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error_tier_two; } + if (true) { stack_pointer += -oparg; goto error_tier_two; } } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; break; } - case BUILD_MAP: { - oparg = CURRENT_OPARG(); + case _BUILD_MAP: { PyObject **values; PyObject *map; - values = stack_pointer - oparg*2; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg*2]; map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + values, 2, + values+1, 2, + oparg); for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error_tier_two; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -oparg*2; goto error_tier_two; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; break; } - case SETUP_ANNOTATIONS: { + case _SETUP_ANNOTATIONS: { int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -1446,13 +1432,13 @@ break; } - case BUILD_CONST_KEY_MAP: { - oparg = CURRENT_OPARG(); + case _BUILD_CONST_KEY_MAP: { PyObject *keys; PyObject **values; PyObject *map; + oparg = CURRENT_OPARG(); keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1460,43 +1446,43 @@ GOTO_ERROR(error); // Pop the keys and values. } map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error_tier_two; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -1 - oparg; goto error_tier_two; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; break; } - case DICT_UPDATE: { - oparg = CURRENT_OPARG(); + case _DICT_UPDATE: { PyObject *update; PyObject *dict; + oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); } Py_DECREF(update); if (true) goto pop_1_error_tier_two; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case DICT_MERGE: { - oparg = CURRENT_OPARG(); + case _DICT_MERGE: { PyObject *update; PyObject *dict; PyObject *callable; + oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; @@ -1506,15 +1492,15 @@ if (true) goto pop_1_error_tier_two; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case MAP_ADD: { - oparg = CURRENT_OPARG(); + case _MAP_ADD: { PyObject *value; PyObject *key; PyObject *dict; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; key = stack_pointer[-2]; dict = stack_pointer[-3 - (oparg - 1)]; @@ -1522,22 +1508,24 @@ /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error_tier_two; - STACK_SHRINK(2); + stack_pointer += -2; break; } - case LOAD_SUPER_ATTR_ATTR: { - oparg = CURRENT_OPARG(); + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + + case _LOAD_SUPER_ATTR_ATTR: { PyObject *self; PyObject *class; PyObject *global_super; PyObject *attr; + oparg = CURRENT_OPARG(); self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; + if (!PyType_Check(class)) goto deoptimize; STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1545,24 +1533,24 @@ Py_DECREF(class); Py_DECREF(self); if (attr == NULL) goto pop_3_error_tier_two; - STACK_SHRINK(2); - stack_pointer[-1] = attr; + stack_pointer[-3] = attr; + stack_pointer += -2 + ((0) ? 1 : 0); break; } - case LOAD_SUPER_ATTR_METHOD: { - oparg = CURRENT_OPARG(); + case _LOAD_SUPER_ATTR_METHOD: { PyObject *self; PyObject *class; PyObject *global_super; PyObject *attr; PyObject *self_or_null; + oparg = CURRENT_OPARG(); self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; + if (!PyType_Check(class)) goto deoptimize; STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1581,17 +1569,17 @@ Py_DECREF(self); self_or_null = NULL; } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; break; } case _LOAD_ATTR: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -1611,7 +1599,7 @@ the second element of the stack to NULL, to signal CALL that it's not a method call. NULL | meth | arg1 | ... | argN - */ + */ Py_DECREF(owner); if (attr == NULL) goto pop_1_error_tier_two; self_or_null = NULL; @@ -1623,9 +1611,9 @@ Py_DECREF(owner); if (attr == NULL) goto pop_1_error_tier_two; } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); break; } @@ -1635,7 +1623,7 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); + if (tp->tp_version_tag != type_version) goto deoptimize; break; } @@ -1645,27 +1633,27 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES); + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; break; } case _LOAD_ATTR_INSTANCE_VALUE: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); + if (attr == NULL) goto deoptimize; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); break; } @@ -1673,18 +1661,18 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE); + if (!PyModule_CheckExact(owner)) goto deoptimize; PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, _CHECK_ATTR_MODULE); + if (dict->ma_keys->dk_version != type_version) goto deoptimize; break; } case _LOAD_ATTR_MODULE: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; @@ -1692,14 +1680,14 @@ assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - DEOPT_IF(attr == NULL, _LOAD_ATTR_MODULE); + if (attr == NULL) goto deoptimize; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); break; } @@ -1708,62 +1696,62 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), _CHECK_ATTR_WITH_HINT); + if (_PyDictOrValues_IsValues(dorv)) goto deoptimize; PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, _CHECK_ATTR_WITH_HINT); + if (dict == NULL) goto deoptimize; assert(PyDict_CheckExact((PyObject *)dict)); break; } case _LOAD_ATTR_WITH_HINT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t hint = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT); + if (hint >= (size_t)dict->ma_keys->dk_nentries) goto deoptimize; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); + if (ep->me_key != name) goto deoptimize; attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); + if (ep->me_key != name) goto deoptimize; attr = ep->me_value; } - DEOPT_IF(attr == NULL, _LOAD_ATTR_WITH_HINT); + if (attr == NULL) goto deoptimize; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); break; } case _LOAD_ATTR_SLOT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); + if (attr == NULL) goto deoptimize; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); break; } @@ -1771,17 +1759,17 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS); + if (!PyType_Check(owner)) goto deoptimize; assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS); + if (((PyTypeObject *)owner)->tp_version_tag != type_version) goto deoptimize; break; } case _LOAD_ATTR_CLASS: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; PyObject *null = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); STAT_INC(LOAD_ATTR, hit); @@ -1789,18 +1777,22 @@ attr = Py_NewRef(descr); null = NULL; Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); break; } + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ + case _GUARD_DORV_VALUES: { PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES); + if (!_PyDictOrValues_IsValues(dorv)) goto deoptimize; break; } @@ -1822,10 +1814,12 @@ Py_DECREF(old_value); } Py_DECREF(owner); - STACK_SHRINK(2); + stack_pointer += -2; break; } + /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + case _STORE_ATTR_SLOT: { PyObject *owner; PyObject *value; @@ -1838,15 +1832,15 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - STACK_SHRINK(2); + stack_pointer += -2; break; } case _COMPARE_OP: { - oparg = CURRENT_OPARG(); PyObject *right; PyObject *left; PyObject *res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; assert((oparg >> 5) <= Py_GE); @@ -1860,20 +1854,20 @@ if (res_bool < 0) goto pop_2_error_tier_two; res = res_bool ? Py_True : Py_False; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case COMPARE_OP_FLOAT: { - oparg = CURRENT_OPARG(); + case _COMPARE_OP_FLOAT: { PyObject *right; PyObject *left; PyObject *res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + if (!PyFloat_CheckExact(left)) goto deoptimize; + if (!PyFloat_CheckExact(right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -1883,22 +1877,22 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case COMPARE_OP_INT: { - oparg = CURRENT_OPARG(); + case _COMPARE_OP_INT: { PyObject *right; PyObject *left; PyObject *res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + if (!PyLong_CheckExact(left)) goto deoptimize; + if (!PyLong_CheckExact(right)) goto deoptimize; + if (!_PyLong_IsCompact((PyLongObject *)left)) goto deoptimize; + if (!_PyLong_IsCompact((PyLongObject *)right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -1910,20 +1904,20 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case COMPARE_OP_STR: { - oparg = CURRENT_OPARG(); + case _COMPARE_OP_STR: { PyObject *right; PyObject *left; PyObject *res; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + if (!PyUnicode_CheckExact(left)) goto deoptimize; + if (!PyUnicode_CheckExact(right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -1934,32 +1928,32 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case IS_OP: { - oparg = CURRENT_OPARG(); + case _IS_OP: { PyObject *right; PyObject *left; PyObject *b; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; break; } - case CONTAINS_OP: { - oparg = CURRENT_OPARG(); + case _CONTAINS_OP: { PyObject *right; PyObject *left; PyObject *b; + oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; int res = PySequence_Contains(right, left); @@ -1967,12 +1961,12 @@ Py_DECREF(right); if (res < 0) goto pop_2_error_tier_two; b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; break; } - case CHECK_EG_MATCH: { + case _CHECK_EG_MATCH: { PyObject *match_type; PyObject *exc_value; PyObject *rest; @@ -1984,18 +1978,15 @@ Py_DECREF(match_type); if (true) goto pop_2_error_tier_two; } - match = NULL; rest = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); + &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); if (res < 0) goto pop_2_error_tier_two; - assert((match == NULL) == (rest == NULL)); if (match == NULL) goto pop_2_error_tier_two; - if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } @@ -2004,7 +1995,7 @@ break; } - case CHECK_EXC_MATCH: { + case _CHECK_EXC_MATCH: { PyObject *right; PyObject *left; PyObject *b; @@ -2012,10 +2003,9 @@ left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error_tier_two; + Py_DECREF(right); + if (true) goto pop_1_error_tier_two; } - int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); b = res ? Py_True : Py_False; @@ -2023,6 +2013,12 @@ break; } + /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + case _IS_NONE: { PyObject *value; PyObject *b; @@ -2038,7 +2034,7 @@ break; } - case GET_LEN: { + case _GET_LEN: { PyObject *obj; PyObject *len_o; obj = stack_pointer[-1]; @@ -2047,17 +2043,17 @@ if (len_i < 0) goto error_tier_two; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = len_o; + stack_pointer[0] = len_o; + stack_pointer += 1; break; } - case MATCH_CLASS: { - oparg = CURRENT_OPARG(); + case _MATCH_CLASS: { PyObject *names; PyObject *type; PyObject *subject; PyObject *attrs; + oparg = CURRENT_OPARG(); names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; @@ -2073,36 +2069,37 @@ } else { if (_PyErr_Occurred(tstate)) goto pop_3_error_tier_two; + // Error! attrs = Py_None; // Failure! } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + stack_pointer[-3] = attrs; + stack_pointer += -2; break; } - case MATCH_MAPPING: { + case _MATCH_MAPPING: { PyObject *subject; PyObject *res; subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case MATCH_SEQUENCE: { + case _MATCH_SEQUENCE: { PyObject *subject; PyObject *res; subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case MATCH_KEYS: { + case _MATCH_KEYS: { PyObject *keys; PyObject *subject; PyObject *values_or_none; @@ -2111,12 +2108,12 @@ // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + stack_pointer[0] = values_or_none; + stack_pointer += 1; break; } - case GET_ITER: { + case _GET_ITER: { PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -2128,7 +2125,7 @@ break; } - case GET_YIELD_FROM_ITER: { + case _GET_YIELD_FROM_ITER: { PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -2160,6 +2157,8 @@ break; } + /* _FOR_ITER is not a viable micro-op for tier 2 */ + case _FOR_ITER_TIER_TWO: { PyObject *iter; PyObject *next; @@ -2177,29 +2176,33 @@ Py_DECREF(iter); STACK_SHRINK(1); /* The translator sets the deopt target just past END_FOR */ - DEOPT_IF(true, _FOR_ITER_TIER_TWO); + if (true) goto deoptimize; } // Common case: no jump, leave it to the code generator - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; break; } + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST); + if (Py_TYPE(iter) != &PyListIter_Type) goto deoptimize; break; } + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + case _GUARD_NOT_EXHAUSTED_LIST: { PyObject *iter; iter = stack_pointer[-1]; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - DEOPT_IF(seq == NULL, _GUARD_NOT_EXHAUSTED_LIST); - DEOPT_IF(it->it_index >= PyList_GET_SIZE(seq), _GUARD_NOT_EXHAUSTED_LIST); + if (seq == NULL) goto deoptimize; + if (it->it_index >= PyList_GET_SIZE(seq)) goto deoptimize; break; } @@ -2213,26 +2216,28 @@ assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; break; } case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE); + if (Py_TYPE(iter) != &PyTupleIter_Type) goto deoptimize; break; } + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + case _GUARD_NOT_EXHAUSTED_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - DEOPT_IF(seq == NULL, _GUARD_NOT_EXHAUSTED_TUPLE); - DEOPT_IF(it->it_index >= PyTuple_GET_SIZE(seq), _GUARD_NOT_EXHAUSTED_TUPLE); + if (seq == NULL) goto deoptimize; + if (it->it_index >= PyTuple_GET_SIZE(seq)) goto deoptimize; break; } @@ -2246,8 +2251,8 @@ assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; break; } @@ -2255,16 +2260,18 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE); + if (Py_TYPE(r) != &PyRangeIter_Type) goto deoptimize; break; } + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + case _GUARD_NOT_EXHAUSTED_RANGE: { PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); - DEOPT_IF(r->len <= 0, _GUARD_NOT_EXHAUSTED_RANGE); + if (r->len <= 0) goto deoptimize; break; } @@ -2280,12 +2287,14 @@ r->len--; next = PyLong_FromLong(value); if (next == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; break; } - case BEFORE_ASYNC_WITH: { + /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ + + case _BEFORE_ASYNC_WITH: { PyObject *mgr; PyObject *exit; PyObject *res; @@ -2319,13 +2328,13 @@ Py_DECREF(exit); if (true) goto pop_1_error_tier_two; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case BEFORE_WITH: { + case _BEFORE_WITH: { PyObject *mgr; PyObject *exit; PyObject *res; @@ -2362,13 +2371,13 @@ Py_DECREF(exit); if (true) goto pop_1_error_tier_two; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case WITH_EXCEPT_START: { + case _WITH_EXCEPT_START: { PyObject *val; PyObject *lasti; PyObject *exit_func; @@ -2383,9 +2392,8 @@ - exit_func: FOURTH = the context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */ + */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); tb = PyException_GetTraceback(val); @@ -2399,14 +2407,14 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case PUSH_EXC_INFO: { + case _PUSH_EXC_INFO: { PyObject *new_exc; PyObject *prev_exc; new_exc = stack_pointer[-1]; @@ -2419,9 +2427,9 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; break; } @@ -2430,7 +2438,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT); + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; break; } @@ -2440,15 +2448,15 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); + if (owner_heap_type->ht_cached_keys->dk_version != keys_version) goto deoptimize; break; } case _LOAD_ATTR_METHOD_WITH_VALUES: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); @@ -2458,17 +2466,17 @@ attr = Py_NewRef(descr); assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += ((1) ? 1 : 0); break; } case _LOAD_ATTR_METHOD_NO_DICT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); @@ -2478,16 +2486,16 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = Py_NewRef(descr); self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += ((1) ? 1 : 0); break; } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); @@ -2496,13 +2504,14 @@ Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; + stack_pointer += ((0) ? 1 : 0); break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); @@ -2512,6 +2521,7 @@ Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; + stack_pointer += ((0) ? 1 : 0); break; } @@ -2522,15 +2532,15 @@ assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, _CHECK_ATTR_METHOD_LAZY_DICT); + if (dict != NULL) goto deoptimize; break; } case _LOAD_ATTR_METHOD_LAZY_DICT: { - oparg = CURRENT_OPARG(); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); @@ -2539,28 +2549,32 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = Py_NewRef(descr); self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += ((1) ? 1 : 0); break; } + /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + + /* _CALL is not a viable micro-op for tier 2 */ + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - oparg = CURRENT_OPARG(); PyObject *null; PyObject *callable; + oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); + if (null != NULL) goto deoptimize; + if (Py_TYPE(callable) != &PyMethod_Type) goto deoptimize; break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - oparg = CURRENT_OPARG(); PyObject *callable; PyObject *func; PyObject *self; + oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; STAT_INC(CALL, hit); self = Py_NewRef(((PyMethodObject *)callable)->im_self); @@ -2574,43 +2588,43 @@ } case _CHECK_PEP_523: { - DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523); + if (tstate->interp->eval_frame) goto deoptimize; break; } case _CHECK_FUNCTION_EXACT_ARGS: { - oparg = CURRENT_OPARG(); PyObject *self_or_null; PyObject *callable; + oparg = CURRENT_OPARG(); self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); + if (!PyFunction_Check(callable)) goto deoptimize; PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); + if (func->func_version != func_version) goto deoptimize; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS); + if (code->co_argcount != oparg + (self_or_null != NULL)) goto deoptimize; break; } case _CHECK_STACK_SPACE: { - oparg = CURRENT_OPARG(); PyObject *callable; + oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE); - DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) goto deoptimize; + if (tstate->py_recursion_remaining <= 1) goto deoptimize; break; } case _INIT_CALL_PY_EXACT_ARGS: { - oparg = CURRENT_OPARG(); PyObject **args; PyObject *self_or_null; PyObject *callable; _PyInterpreterFrame *new_frame; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int argcount = oparg; @@ -2624,129 +2638,130 @@ for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = args[i]; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = (PyObject *)new_frame; + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; break; } case _PUSH_FRAME: { _PyInterpreterFrame *new_frame; new_frame = (_PyInterpreterFrame *)stack_pointer[-1]; - STACK_SHRINK(1); // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif + #endif + stack_pointer += ((0) ? 1 : 0); break; } - case CALL_TYPE_1: { - oparg = CURRENT_OPARG(); + /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ + + case _CALL_TYPE_1: { PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); + if (null != NULL) goto deoptimize; PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + if (callable != (PyObject *)&PyType_Type) goto deoptimize; STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; break; } - case CALL_STR_1: { - oparg = CURRENT_OPARG(); + case _CALL_STR_1: { PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + if (null != NULL) goto deoptimize; + if (callable != (PyObject *)&PyUnicode_Type) goto deoptimize; STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_TUPLE_1: { - oparg = CURRENT_OPARG(); + case _CALL_TUPLE_1: { PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + if (null != NULL) goto deoptimize; + if (callable != (PyObject *)&PyTuple_Type) goto deoptimize; STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case EXIT_INIT_CHECK: { + /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + + case _EXIT_INIT_CHECK: { PyObject *should_be_none; should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); GOTO_ERROR(error); } - STACK_SHRINK(1); + stack_pointer += -1; break; } - case CALL_BUILTIN_CLASS: { - oparg = CURRENT_OPARG(); + case _CALL_BUILTIN_CLASS: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -2754,9 +2769,9 @@ args--; total_args++; } - DEOPT_IF(!PyType_Check(callable), CALL); + if (!PyType_Check(callable)) goto deoptimize; PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + if (tp->tp_vectorcall == NULL) goto deoptimize; STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -2764,21 +2779,20 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_BUILTIN_O: { - oparg = CURRENT_OPARG(); + case _CALL_BUILTIN_O: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ @@ -2787,9 +2801,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + if (total_args != 1) goto deoptimize; + if (!PyCFunction_CheckExact(callable)) goto deoptimize; + if (PyCFunction_GET_FLAGS(callable) != METH_O) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall @@ -2801,24 +2815,22 @@ res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_BUILTIN_FAST: { - oparg = CURRENT_OPARG(); + case _CALL_BUILTIN_FAST: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ @@ -2827,8 +2839,8 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + if (!PyCFunction_CheckExact(callable)) goto deoptimize; + if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -2837,32 +2849,30 @@ args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_BUILTIN_FAST_WITH_KEYWORDS: { - oparg = CURRENT_OPARG(); + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ @@ -2871,36 +2881,34 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + if (!PyCFunction_CheckExact(callable)) goto deoptimize; + if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) goto deoptimize; STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_LEN: { - oparg = CURRENT_OPARG(); + case _CALL_LEN: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ @@ -2909,9 +2917,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + if (total_args != 1) goto deoptimize; PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); + if (callable != interp->callable_cache.len) goto deoptimize; STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); @@ -2920,23 +2928,21 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; break; } - case CALL_ISINSTANCE: { - oparg = CURRENT_OPARG(); + case _CALL_ISINSTANCE: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ @@ -2945,9 +2951,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + if (total_args != 2) goto deoptimize; PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + if (callable != interp->callable_cache.isinstance) goto deoptimize; STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; @@ -2957,24 +2963,22 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; break; } - case CALL_METHOD_DESCRIPTOR_O: { - oparg = CURRENT_OPARG(); + case _CALL_METHOD_DESCRIPTOR_O: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -2983,13 +2987,13 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (total_args != 2) goto deoptimize; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + if (meth->ml_flags != METH_O) goto deoptimize; PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3003,21 +3007,20 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - oparg = CURRENT_OPARG(); + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -3026,39 +3029,37 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) goto deoptimize; PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + if (!Py_IS_TYPE(self, d_type)) goto deoptimize; STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; res = cfunc(self, args + 1, nargs, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_METHOD_DESCRIPTOR_NOARGS: { - oparg = CURRENT_OPARG(); + case _CALL_METHOD_DESCRIPTOR_NOARGS: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); @@ -3067,13 +3068,13 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + if (total_args != 1) goto deoptimize; PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; + if (meth->ml_flags != METH_NOARGS) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3086,21 +3087,20 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case CALL_METHOD_DESCRIPTOR_FAST: { - oparg = CURRENT_OPARG(); + case _CALL_METHOD_DESCRIPTOR_FAST: { PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -3110,14 +3110,14 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + if (meth->ml_flags != METH_FASTCALL) goto deoptimize; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; STAT_INC(CALL, hit); _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3126,93 +3126,96 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); break; } - case MAKE_FUNCTION: { + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + + /* _CALL_KW is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + case _MAKE_FUNCTION: { PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - + PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { GOTO_ERROR(error); } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; break; } - case SET_FUNCTION_ATTRIBUTE: { - oparg = CURRENT_OPARG(); + case _SET_FUNCTION_ATTRIBUTE: { PyObject *func; PyObject *attr; + oparg = CURRENT_OPARG(); func = stack_pointer[-1]; attr = stack_pointer[-2]; assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; default: - Py_UNREACHABLE(); + Py_UNREACHABLE(); } - STACK_SHRINK(1); - stack_pointer[-1] = func; + stack_pointer[-2] = func; + stack_pointer += -1; break; } - case BUILD_SLICE: { - oparg = CURRENT_OPARG(); + case _BUILD_SLICE: { PyObject *step = NULL; PyObject *stop; PyObject *start; PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; + oparg = CURRENT_OPARG(); + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error_tier_two; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error_tier_two; } + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); break; } - case CONVERT_VALUE: { - oparg = CURRENT_OPARG(); + case _CONVERT_VALUE: { PyObject *value; PyObject *result; + oparg = CURRENT_OPARG(); value = stack_pointer[-1]; convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); @@ -3224,7 +3227,7 @@ break; } - case FORMAT_SIMPLE: { + case _FORMAT_SIMPLE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -3242,7 +3245,7 @@ break; } - case FORMAT_WITH_SPEC: { + case _FORMAT_WITH_SPEC: { PyObject *fmt_spec; PyObject *value; PyObject *res; @@ -3252,28 +3255,28 @@ Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case COPY: { - oparg = CURRENT_OPARG(); + case _COPY: { PyObject *bottom; PyObject *top; + oparg = CURRENT_OPARG(); bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; + stack_pointer[0] = top; + stack_pointer += 1; break; } case _BINARY_OP: { - oparg = CURRENT_OPARG(); PyObject *rhs; PyObject *lhs; PyObject *res; + oparg = CURRENT_OPARG(); rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; assert(_PyEval_BinaryOps[oparg]); @@ -3281,15 +3284,15 @@ Py_DECREF(lhs); Py_DECREF(rhs); if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case SWAP: { - oparg = CURRENT_OPARG(); + case _SWAP: { PyObject *top; PyObject *bottom; + oparg = CURRENT_OPARG(); top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); @@ -3298,38 +3301,52 @@ break; } + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + case _GUARD_IS_TRUE_POP: { PyObject *flag; flag = stack_pointer[-1]; - DEOPT_IF(Py_IsFalse(flag), _GUARD_IS_TRUE_POP); + if (Py_IsFalse(flag)) goto deoptimize; assert(Py_IsTrue(flag)); - STACK_SHRINK(1); + stack_pointer += -1; break; } case _GUARD_IS_FALSE_POP: { PyObject *flag; flag = stack_pointer[-1]; - DEOPT_IF(Py_IsTrue(flag), _GUARD_IS_FALSE_POP); + if (Py_IsTrue(flag)) goto deoptimize; assert(Py_IsFalse(flag)); - STACK_SHRINK(1); + stack_pointer += -1; break; } case _GUARD_IS_NONE_POP: { PyObject *val; val = stack_pointer[-1]; - DEOPT_IF(!Py_IsNone(val), _GUARD_IS_NONE_POP); - STACK_SHRINK(1); + if (!Py_IsNone(val)) goto deoptimize; + stack_pointer += -1; break; } case _GUARD_IS_NOT_NONE_POP: { PyObject *val; val = stack_pointer[-1]; - DEOPT_IF(Py_IsNone(val), _GUARD_IS_NOT_NONE_POP); + if (Py_IsNone(val)) goto deoptimize; Py_DECREF(val); - STACK_SHRINK(1); + stack_pointer += -1; break; } @@ -3360,13 +3377,13 @@ case _EXIT_TRACE: { TIER_TWO_ONLY - GOTO_TIER_ONE(); + if (1) goto deoptimize; break; } case _INSERT: { - oparg = CURRENT_OPARG(); PyObject *top; + oparg = CURRENT_OPARG(); top = stack_pointer[-1]; // Inserts TOS at position specified by oparg; memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); @@ -3376,7 +3393,7 @@ case _CHECK_VALIDITY: { TIER_TWO_ONLY - DEOPT_IF(!current_executor->base.vm_data.valid, _CHECK_VALIDITY); + if (!current_executor->base.vm_data.valid) goto deoptimize; break; } diff --git a/Python/fileutils.c b/Python/fileutils.c index 649b188b5167d0..882d3299575cf3 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2878,9 +2878,9 @@ _Py_GetLocaleconvNumeric(struct lconv *lc, * non-opened fd in the middle. * 2b. If fdwalk(3) isn't available, just do a plain close(2) loop. */ -#ifdef __FreeBSD__ +#ifdef HAVE_CLOSEFROM # define USE_CLOSEFROM -#endif /* __FreeBSD__ */ +#endif /* HAVE_CLOSEFROM */ #ifdef HAVE_FDWALK # define USE_FDWALK @@ -2922,7 +2922,7 @@ _Py_closerange(int first, int last) #ifdef USE_CLOSEFROM if (last >= sysconf(_SC_OPEN_MAX)) { /* Any errors encountered while closing file descriptors are ignored */ - closefrom(first); + (void)closefrom(first); } else #endif /* USE_CLOSEFROM */ @@ -2943,3 +2943,27 @@ _Py_closerange(int first, int last) #endif /* USE_FDWALK */ _Py_END_SUPPRESS_IPH } + + +#ifndef MS_WINDOWS +// Ticks per second used by clock() and times() functions. +// See os.times() and time.process_time() implementations. +int +_Py_GetTicksPerSecond(long *ticks_per_second) +{ +#if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) + long value = sysconf(_SC_CLK_TCK); + if (value < 1) { + return -1; + } + *ticks_per_second = value; +#elif defined(HZ) + assert(HZ >= 1); + *ticks_per_second = HZ; +#else + // Magic fallback value; may be bogus + *ticks_per_second = 60; +#endif + return 0; +} +#endif diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 87401e14f97f02..5bb11980b8ca37 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -97,6 +97,7 @@ static const jump_target_label NO_LABEL = {-1}; static inline int is_block_push(cfg_instr *i) { + assert(OPCODE_HAS_ARG(i->i_opcode) || !IS_BLOCK_PUSH_OPCODE(i->i_opcode)); return IS_BLOCK_PUSH_OPCODE(i->i_opcode); } @@ -448,6 +449,15 @@ _PyCfgBuilder_Addop(cfg_builder *g, int opcode, int oparg, location loc) } +static basicblock * +next_nonempty_block(basicblock *b) +{ + while (b && b->b_iused == 0) { + b = b->b_next; + } + return b; +} + /***** debugging helpers *****/ #ifndef NDEBUG @@ -463,24 +473,16 @@ no_redundant_nops(cfg_builder *g) { return true; } -static bool -no_empty_basic_blocks(cfg_builder *g) { - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - if (b->b_iused == 0) { - return false; - } - } - return true; -} - static bool no_redundant_jumps(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { cfg_instr *last = basicblock_last_instr(b); if (last != NULL) { if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { - assert(last->i_target != b->b_next); - if (last->i_target == b->b_next) { + basicblock *next = next_nonempty_block(b->b_next); + basicblock *jump_target = next_nonempty_block(last->i_target); + assert(jump_target != next); + if (jump_target == next) { return false; } } @@ -647,7 +649,7 @@ mark_except_handlers(basicblock *entryblock) { struct _PyCfgExceptStack { - basicblock *handlers[CO_MAXBLOCKS+1]; + basicblock *handlers[CO_MAXBLOCKS+2]; int depth; }; @@ -660,6 +662,7 @@ push_except_block(struct _PyCfgExceptStack *stack, cfg_instr *setup) { if (opcode == SETUP_WITH || opcode == SETUP_CLEANUP) { target->b_preserve_lasti = 1; } + assert(stack->depth <= CO_MAXBLOCKS); stack->handlers[++stack->depth] = target; return target; } @@ -959,42 +962,6 @@ mark_reachable(basicblock *entryblock) { return SUCCESS; } -static void -eliminate_empty_basic_blocks(cfg_builder *g) { - /* Eliminate empty blocks */ - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - basicblock *next = b->b_next; - while (next && next->b_iused == 0) { - next = next->b_next; - } - b->b_next = next; - } - while(g->g_entryblock && g->g_entryblock->b_iused == 0) { - g->g_entryblock = g->g_entryblock->b_next; - } - int next_lbl = get_max_label(g->g_entryblock) + 1; - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - assert(b->b_iused > 0); - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *instr = &b->b_instr[i]; - if (HAS_TARGET(instr->i_opcode)) { - basicblock *target = instr->i_target; - while (target->b_iused == 0) { - target = target->b_next; - } - if (instr->i_target != target) { - if (!IS_LABEL(target->b_label)) { - target->b_label.id = next_lbl++; - } - instr->i_target = target; - instr->i_oparg = target->b_label.id; - } - assert(instr->i_target && instr->i_target->b_iused > 0); - } - } - } -} - static int remove_redundant_nops(basicblock *bb) { /* Remove NOPs when legal to do so. */ @@ -1023,10 +990,7 @@ remove_redundant_nops(basicblock *bb) { } } else { - basicblock* next = bb->b_next; - while (next && next->b_iused == 0) { - next = next->b_next; - } + basicblock *next = next_nonempty_block(bb->b_next); /* or if last instruction in BB and next BB has same line number */ if (next) { location next_loc = NO_LOCATION; @@ -1109,19 +1073,27 @@ remove_redundant_jumps(cfg_builder *g) { * of that jump. If it is, then the jump instruction is redundant and * can be deleted. */ - assert(no_empty_basic_blocks(g)); + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { cfg_instr *last = basicblock_last_instr(b); - assert(last != NULL); + if (last == NULL) { + continue; + } assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { - if (last->i_target == NULL) { + basicblock* jump_target = next_nonempty_block(last->i_target); + if (jump_target == NULL) { PyErr_SetString(PyExc_SystemError, "jump with NULL target"); return ERROR; } - if (last->i_target == b->b_next) { - assert(b->b_next->b_iused); - INSTR_SET_OP0(last, NOP); + basicblock *next = next_nonempty_block(b->b_next); + if (jump_target == next) { + if (last->i_loc.lineno == NO_LOCATION.lineno) { + b->b_iused--; + } + else { + INSTR_SET_OP0(last, NOP); + } } } } @@ -1732,11 +1704,9 @@ optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache) { assert(PyDict_CheckExact(const_cache)); RETURN_IF_ERROR(check_cfg(g)); - eliminate_empty_basic_blocks(g); for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { RETURN_IF_ERROR(inline_small_exit_blocks(b)); } - assert(no_empty_basic_blocks(g)); for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { RETURN_IF_ERROR(optimize_basic_block(const_cache, b, consts)); assert(b->b_predecessors == 0); @@ -1751,14 +1721,21 @@ optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache) for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { if (b->b_predecessors == 0) { b->b_iused = 0; + b->b_except_handler = 0; } } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { remove_redundant_nops(b); } - eliminate_empty_basic_blocks(g); - assert(no_redundant_nops(g)); RETURN_IF_ERROR(remove_redundant_jumps(g)); + + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { + remove_redundant_nops(b); + } + + RETURN_IF_ERROR(remove_redundant_jumps(g)); + + assert(no_redundant_jumps(g)); return SUCCESS; } @@ -1808,7 +1785,6 @@ insert_superinstructions(cfg_builder *g) for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { remove_redundant_nops(b); } - eliminate_empty_basic_blocks(g); assert(no_redundant_nops(g)); } @@ -2239,15 +2215,14 @@ convert_pseudo_ops(basicblock *entryblock) for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { - assert(SAME_OPCODE_METADATA(instr->i_opcode, NOP)); INSTR_SET_OP0(instr, NOP); } else if (instr->i_opcode == LOAD_CLOSURE) { - assert(SAME_OPCODE_METADATA(LOAD_CLOSURE, LOAD_FAST)); + assert(is_pseudo_target(LOAD_CLOSURE, LOAD_FAST)); instr->i_opcode = LOAD_FAST; } else if (instr->i_opcode == STORE_FAST_MAYBE_NULL) { - assert(SAME_OPCODE_METADATA(STORE_FAST_MAYBE_NULL, STORE_FAST)); + assert(is_pseudo_target(STORE_FAST_MAYBE_NULL, STORE_FAST)); instr->i_opcode = STORE_FAST; } } @@ -2283,8 +2258,6 @@ is_exit_without_lineno(basicblock *b) { static int duplicate_exits_without_lineno(cfg_builder *g) { - assert(no_empty_basic_blocks(g)); - int next_lbl = get_max_label(g->g_entryblock) + 1; /* Copy all exit blocks without line number that are targets of a jump. @@ -2292,9 +2265,11 @@ duplicate_exits_without_lineno(cfg_builder *g) basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { cfg_instr *last = basicblock_last_instr(b); - assert(last != NULL); + if (last == NULL) { + continue; + } if (is_jump(last)) { - basicblock *target = last->i_target; + basicblock *target = next_nonempty_block(last->i_target); if (is_exit_without_lineno(target) && target->b_predecessors > 1) { basicblock *new_target = copy_basicblock(g, target); if (new_target == NULL) { @@ -2351,9 +2326,10 @@ propagate_line_numbers(basicblock *entryblock) { } } if (BB_HAS_FALLTHROUGH(b) && b->b_next->b_predecessors == 1) { - assert(b->b_next->b_iused); - if (b->b_next->b_instr[0].i_loc.lineno < 0) { - b->b_next->b_instr[0].i_loc = prev_location; + if (b->b_next->b_iused > 0) { + if (b->b_next->b_instr[0].i_loc.lineno < 0) { + b->b_next->b_instr[0].i_loc = prev_location; + } } } if (is_jump(last)) { diff --git a/Python/frozen.c b/Python/frozen.c index 0fb38a11902f35..77f51a7f750965 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -80,7 +80,6 @@ extern PyObject *_Py_get__sitebuiltins_toplevel(void); extern PyObject *_Py_get_genericpath_toplevel(void); extern PyObject *_Py_get_ntpath_toplevel(void); extern PyObject *_Py_get_posixpath_toplevel(void); -extern PyObject *_Py_get_posixpath_toplevel(void); extern PyObject *_Py_get_os_toplevel(void); extern PyObject *_Py_get_site_toplevel(void); extern PyObject *_Py_get_stat_toplevel(void); @@ -88,13 +87,8 @@ extern PyObject *_Py_get_importlib_util_toplevel(void); extern PyObject *_Py_get_importlib_machinery_toplevel(void); extern PyObject *_Py_get_runpy_toplevel(void); extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___phello___toplevel(void); extern PyObject *_Py_get___phello___toplevel(void); extern PyObject *_Py_get___phello___ham_toplevel(void); -extern PyObject *_Py_get___phello___ham_toplevel(void); extern PyObject *_Py_get___phello___ham_eggs_toplevel(void); extern PyObject *_Py_get___phello___spam_toplevel(void); extern PyObject *_Py_get_frozen_only_toplevel(void); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dedd793111b7ff..8226d827cde514 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1,4 +1,4 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/tier1_generator.py // from: // Python/bytecodes.c // Do not edit! @@ -8,6 +8,7 @@ #endif #define TIER_ONE 1 + TARGET(BEFORE_ASYNC_WITH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -45,9 +46,9 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -91,9 +92,9 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -103,7 +104,6 @@ INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs; PyObject *lhs; PyObject *res; @@ -133,8 +133,8 @@ Py_DECREF(rhs); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -142,6 +142,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -152,16 +153,17 @@ DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } + /* Skip 1 cache entry */ // _BINARY_OP_ADD_FLOAT { STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -169,6 +171,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -179,6 +182,7 @@ DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } + /* Skip 1 cache entry */ // _BINARY_OP_ADD_INT { STAT_INC(BINARY_OP, hit); @@ -187,8 +191,8 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -196,6 +200,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -206,6 +211,7 @@ DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } + /* Skip 1 cache entry */ // _BINARY_OP_ADD_UNICODE { STAT_INC(BINARY_OP, hit); @@ -214,8 +220,8 @@ _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -223,6 +229,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; // _GUARD_BOTH_UNICODE @@ -232,8 +239,10 @@ DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } + /* Skip 1 cache entry */ // _BINARY_OP_INPLACE_ADD_UNICODE { + TIER_ONE_ONLY assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left, BINARY_OP); @@ -258,7 +267,7 @@ assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -266,6 +275,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -276,16 +286,17 @@ DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } + /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_FLOAT { STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -293,6 +304,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -303,6 +315,7 @@ DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } + /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_INT { STAT_INC(BINARY_OP, hit); @@ -311,8 +324,8 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -320,6 +333,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -330,16 +344,17 @@ DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } + /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_FLOAT { STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -347,6 +362,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -357,6 +373,7 @@ DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } + /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_INT { STAT_INC(BINARY_OP, hit); @@ -365,8 +382,8 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -393,8 +410,8 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = res; + stack_pointer[-3] = res; + stack_pointer += -2; DISPATCH(); } @@ -404,7 +421,6 @@ INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *res; @@ -431,8 +447,8 @@ Py_DECREF(sub); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -440,9 +456,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *dict; PyObject *res; + /* Skip 1 cache entry */ sub = stack_pointer[-1]; dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); @@ -454,8 +472,9 @@ Py_DECREF(dict); Py_DECREF(sub); if (rc <= 0) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -463,8 +482,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; + /* Skip 1 cache entry */ sub = stack_pointer[-1]; container = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); @@ -494,14 +515,15 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *list; PyObject *res; + /* Skip 1 cache entry */ sub = stack_pointer[-1]; list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -512,8 +534,8 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -521,9 +543,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *str; PyObject *res; + /* Skip 1 cache entry */ sub = stack_pointer[-1]; str = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -538,8 +562,8 @@ res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(str); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -547,14 +571,15 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *tuple; PyObject *res; + /* Skip 1 cache entry */ sub = stack_pointer[-1]; tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -565,8 +590,8 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -578,7 +603,7 @@ PyObject **values; PyObject *map; keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -586,15 +611,15 @@ GOTO_ERROR(error); // Pop the keys and values. } map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; DISPATCH(); } @@ -604,12 +629,11 @@ INSTRUCTION_STATS(BUILD_LIST); PyObject **values; PyObject *list; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -619,18 +643,17 @@ INSTRUCTION_STATS(BUILD_MAP); PyObject **values; PyObject *map; - values = stack_pointer - oparg*2; + values = &stack_pointer[-oparg*2]; map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + values, 2, + values+1, 2, + oparg); for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; DISPATCH(); } @@ -640,24 +663,23 @@ INSTRUCTION_STATS(BUILD_SET); PyObject **values; PyObject *set; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + GOTO_ERROR(error); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; if (err == 0) - err = PySet_Add(set, item); + err = PySet_Add(set, item); Py_DECREF(item); } if (err != 0) { Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error; } + if (true) { stack_pointer += -oparg; goto error; } } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -669,17 +691,16 @@ PyObject *stop; PyObject *start; PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; } + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); DISPATCH(); } @@ -689,15 +710,14 @@ INSTRUCTION_STATS(BUILD_STRING); PyObject **pieces; PyObject *str; - pieces = stack_pointer - oparg; + pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -707,12 +727,11 @@ INSTRUCTION_STATS(BUILD_TUPLE); PyObject **values; PyObject *tup; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -720,6 +739,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); + TIER_ONE_ONLY assert(0 && "Executing a cache."); Py_UNREACHABLE(); } @@ -730,13 +750,12 @@ INSTRUCTION_STATS(CALL); PREDICTED(CALL); _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; // _SPECIALIZE_CALL - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { @@ -752,6 +771,7 @@ DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } + /* Skip 2 cache entries */ // _CALL { // oparg counts all of the args, but *not* self: @@ -793,12 +813,12 @@ } /* Callable is not a normal Python function */ res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -818,11 +838,10 @@ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -831,10 +850,13 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* This instruction does the following: @@ -890,13 +912,15 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject *null; PyObject *callable; + PyObject *func; PyObject *self; PyObject *self_or_null; - PyObject *func; PyObject **args; _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ // _CHECK_PEP_523 { DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -936,7 +960,8 @@ DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; { int argcount = oparg; if (self_or_null != NULL) { @@ -960,26 +985,26 @@ #endif } // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif + #endif } + stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -987,11 +1012,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1009,10 +1037,9 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1021,11 +1048,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ @@ -1044,21 +1074,19 @@ args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1067,11 +1095,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ @@ -1085,20 +1116,18 @@ STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1107,11 +1136,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ @@ -1134,13 +1166,11 @@ res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1155,9 +1185,9 @@ PyObject *callargs; PyObject *func; PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } - callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; - func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)]; + if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -1177,7 +1207,7 @@ !PyFunction_Check(func) && !PyMethod_Check(func) ) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; + PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); @@ -1205,10 +1235,9 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -1224,10 +1253,9 @@ Py_DECREF(callargs); Py_XDECREF(kwargs); assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - STACK_SHRINK(((oparg & 1) ? 1 : 0)); - STACK_SHRINK(2); - stack_pointer[-1] = result; + if (result == NULL) { stack_pointer += -3 - (oparg & 1); goto error; } + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1261,8 +1289,8 @@ Py_DECREF(value2); Py_DECREF(value1); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -1270,11 +1298,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ @@ -1295,14 +1326,12 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } @@ -1318,7 +1347,7 @@ PyObject *callable; PyObject *res; kwnames = stack_pointer[-1]; - args = stack_pointer - 1 - oparg; + args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; // oparg counts all of the args, but *not* self: @@ -1363,12 +1392,12 @@ } /* Callable is not a normal Python function */ res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -1389,10 +1418,9 @@ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_3_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(2); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1401,11 +1429,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ @@ -1425,13 +1456,11 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } @@ -1439,12 +1468,16 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self; PyObject *callable; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + TIER_ONE_ONLY assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); @@ -1467,11 +1500,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1488,7 +1524,7 @@ DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1497,10 +1533,9 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1509,11 +1544,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1531,19 +1569,17 @@ STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; res = cfunc(self, args + 1, nargs, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1552,11 +1588,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); @@ -1584,10 +1623,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1596,11 +1634,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1629,10 +1670,9 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1641,10 +1681,12 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject *self_or_null; PyObject *callable; PyObject **args; _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ // _CHECK_PEP_523 { DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -1668,7 +1710,8 @@ DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; { int argcount = oparg; if (self_or_null != NULL) { @@ -1692,26 +1735,26 @@ #endif } // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif + #endif } + stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -1719,10 +1762,12 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); @@ -1763,11 +1808,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1778,10 +1826,9 @@ res = PyObject_Str(arg); Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1790,11 +1837,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1805,10 +1855,9 @@ res = PySequence_Tuple(arg); Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1817,11 +1866,14 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1832,9 +1884,8 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } @@ -1853,18 +1904,15 @@ Py_DECREF(match_type); if (true) goto pop_2_error; } - match = NULL; rest = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); + &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); if (res < 0) goto pop_2_error; - assert((match == NULL) == (rest == NULL)); if (match == NULL) goto pop_2_error; - if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } @@ -1884,10 +1932,9 @@ left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; + Py_DECREF(right); + if (true) goto pop_1_error; } - int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); b = res ? Py_True : Py_False; @@ -1922,9 +1969,9 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - STACK_SHRINK(1); - stack_pointer[-2] = none; - stack_pointer[-1] = value; + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } @@ -1934,7 +1981,6 @@ INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -1968,8 +2014,8 @@ res = res_bool ? Py_True : Py_False; } } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -1977,9 +2023,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; + /* Skip 1 cache entry */ right = stack_pointer[-1]; left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); @@ -1993,8 +2041,8 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2002,9 +2050,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; + /* Skip 1 cache entry */ right = stack_pointer[-1]; left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); @@ -2022,8 +2072,8 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2031,9 +2081,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; + /* Skip 1 cache entry */ right = stack_pointer[-1]; left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); @@ -2048,8 +2100,8 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2067,8 +2119,8 @@ Py_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } @@ -2098,8 +2150,8 @@ bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; + stack_pointer[0] = top; + stack_pointer += 1; DISPATCH(); } @@ -2130,7 +2182,7 @@ int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2172,7 +2224,7 @@ if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } GOTO_ERROR(error); } @@ -2195,8 +2247,8 @@ // Can't use ERROR_IF here. if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + NAME_ERROR_MSG, + name); GOTO_ERROR(error); } DISPATCH(); @@ -2215,7 +2267,7 @@ Py_DECREF(container); Py_DECREF(sub); if (err) goto pop_2_error; - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2235,7 +2287,7 @@ if (true) goto pop_1_error; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2250,14 +2302,14 @@ if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); } Py_DECREF(update); if (true) goto pop_1_error; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2281,7 +2333,7 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2290,17 +2342,17 @@ next_instr += 1; INSTRUCTION_STATS(END_FOR); PyObject *value; - // POP_TOP + // _POP_TOP value = stack_pointer[-1]; { Py_DECREF(value); } - // POP_TOP + // _POP_TOP value = stack_pointer[-2]; { Py_DECREF(value); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2313,34 +2365,42 @@ value = stack_pointer[-1]; receiver = stack_pointer[-2]; Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } TARGET(ENTER_EXECUTOR) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); TIER_ONE_ONLY CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); - frame->instr_ptr = next_instr; - Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { - current_executor = (_PyUOpExecutorObject *)executor; - GOTO_TIER_TWO(); - } - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { + if (executor->vm_data.valid) { + Py_INCREF(executor); + if (executor->execute == _PyUOpExecute) { + current_executor = (_PyUOpExecutorObject *)executor; + GOTO_TIER_TWO(); + } + next_instr = executor->execute(executor, frame, stack_pointer); frame = tstate->current_frame; - goto resume_with_error; + if (next_instr == NULL) { + goto resume_with_error; + } + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + code->co_executors->executors[oparg & 255] = NULL; + opcode = this_instr->op.code = executor->vm_data.opcode; + this_instr->op.arg = executor->vm_data.oparg; + oparg = (oparg & (~255)) | executor->vm_data.oparg; + Py_DECREF(executor); + next_instr = this_instr; + DISPATCH_GOTO(); } - goto enter_tier_one; + DISPATCH(); } TARGET(EXIT_INIT_CHECK) { @@ -2352,11 +2412,11 @@ assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); GOTO_ERROR(error); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2364,6 +2424,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); + TIER_ONE_ONLY assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -2405,8 +2466,8 @@ Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2416,7 +2477,6 @@ INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; // _SPECIALIZE_FOR_ITER @@ -2448,7 +2508,7 @@ } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2457,8 +2517,8 @@ } // Common case: no jump, leave it to the code generator } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2466,7 +2526,9 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; + /* Skip 1 cache entry */ iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; @@ -2489,8 +2551,10 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_LIST iter = stack_pointer[-1]; { @@ -2523,8 +2587,8 @@ assert(it->it_index < PyList_GET_SIZE(seq)); next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2532,8 +2596,10 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_RANGE iter = stack_pointer[-1]; { @@ -2564,8 +2630,8 @@ next = PyLong_FromLong(value); if (next == NULL) goto error; } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2573,8 +2639,10 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { @@ -2607,8 +2675,8 @@ assert(it->it_index < PyTuple_GET_SIZE(seq)); next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2621,11 +2689,9 @@ obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " @@ -2634,14 +2700,11 @@ Py_DECREF(obj); if (true) goto pop_1_error; } - iter = (*getter)(obj); Py_DECREF(obj); if (iter == NULL) goto pop_1_error; - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", @@ -2663,7 +2726,6 @@ unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { @@ -2673,7 +2735,6 @@ if (type->tp_as_async != NULL){ getter = type->tp_as_async->am_anext; } - if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { @@ -2687,7 +2748,6 @@ type->tp_name); GOTO_ERROR(error); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { _PyErr_FormatFromCause( @@ -2695,15 +2755,14 @@ "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); GOTO_ERROR(error); } else { Py_DECREF(next_iter); } } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + stack_pointer[0] = awaitable; + stack_pointer += 1; DISPATCH(); } @@ -2715,13 +2774,10 @@ PyObject *iter; iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { @@ -2735,7 +2791,6 @@ /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error; stack_pointer[-1] = iter; DISPATCH(); @@ -2768,8 +2823,8 @@ if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = len_o; + stack_pointer[0] = len_o; + stack_pointer += 1; DISPATCH(); } @@ -2819,8 +2874,8 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -2839,8 +2894,8 @@ Py_DECREF(level); Py_DECREF(fromlist); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2848,14 +2903,15 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); + /* Skip 3 cache entries */ int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); if (err) goto error; INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(CALL); @@ -2876,10 +2932,10 @@ int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 3); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); + : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } @@ -2904,7 +2960,7 @@ } Py_DECREF(receiver); Py_DECREF(value); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2925,8 +2981,8 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } @@ -2934,6 +2990,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + /* Skip 1 cache entry */ _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2981,6 +3038,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ CHECK_EVAL_BREAKER(); INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); @@ -2998,6 +3056,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + /* Skip 1 cache entry */ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); @@ -3008,6 +3067,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + /* Skip 1 cache entry */ PyObject *cond = POP(); assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); @@ -3023,6 +3083,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + /* Skip 1 cache entry */ PyObject *value = POP(); int flag = Py_IsNone(value); int offset; @@ -3044,6 +3105,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + /* Skip 1 cache entry */ PyObject *value = POP(); int offset; int nflag = Py_IsNone(value); @@ -3065,6 +3127,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + /* Skip 1 cache entry */ PyObject *cond = POP(); assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); @@ -3094,7 +3157,7 @@ } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); + tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; if (frame->instr_ptr != this_instr) { @@ -3112,8 +3175,8 @@ INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); if (err) GOTO_ERROR(error); Py_INCREF(retval); assert(EMPTY()); @@ -3136,8 +3199,8 @@ PyObject *retval; retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); if (err) GOTO_ERROR(error); STACK_SHRINK(1); assert(EMPTY()); @@ -3167,8 +3230,8 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); if (err) GOTO_ERROR(error); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -3189,6 +3252,7 @@ INSTRUCTION_STATS(INTERPRETER_EXIT); PyObject *retval; retval = stack_pointer[-1]; + TIER_ONE_ONLY assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ @@ -3211,8 +3275,8 @@ Py_DECREF(left); Py_DECREF(right); b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } @@ -3220,6 +3284,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); + /* Skip 1 cache entry */ CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); @@ -3260,6 +3325,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + TIER_ONE_ONLY /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3273,6 +3339,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); + TIER_ONE_ONLY JUMPBY(oparg); DISPATCH(); } @@ -3286,7 +3353,7 @@ v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -3301,19 +3368,19 @@ PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); Py_DECREF(iterable); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -3323,8 +3390,8 @@ INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); PyObject *value; value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3334,7 +3401,6 @@ INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); _Py_CODEUNIT *this_instr = next_instr - 10; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; @@ -3354,6 +3420,7 @@ DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } + /* Skip 8 cache entries */ // _LOAD_ATTR { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); @@ -3374,7 +3441,7 @@ the second element of the stack to NULL, to signal CALL that it's not a method call. NULL | meth | arg1 | ... | argN - */ + */ Py_DECREF(owner); if (attr == NULL) goto pop_1_error; self_or_null = NULL; @@ -3387,9 +3454,9 @@ if (attr == NULL) goto pop_1_error; } } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); DISPATCH(); } @@ -3397,9 +3464,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; { @@ -3408,6 +3477,7 @@ assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3417,9 +3487,9 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } @@ -3427,7 +3497,9 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; + /* Skip 1 cache entry */ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); uint32_t func_version = read_u32(&this_instr[4].cache); @@ -3445,7 +3517,6 @@ assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); @@ -3461,9 +3532,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3490,9 +3563,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } @@ -3500,9 +3574,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3519,6 +3595,7 @@ /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_LAZY_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3529,9 +3606,9 @@ attr = Py_NewRef(descr); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += ((1) ? 1 : 0); DISPATCH(); } @@ -3539,9 +3616,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3550,6 +3629,7 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3561,9 +3641,9 @@ attr = Py_NewRef(descr); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += ((1) ? 1 : 0); DISPATCH(); } @@ -3571,9 +3651,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3606,9 +3688,9 @@ assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += ((1) ? 1 : 0); DISPATCH(); } @@ -3616,9 +3698,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { @@ -3642,9 +3726,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } @@ -3652,8 +3737,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3662,6 +3749,7 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3673,6 +3761,7 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; + stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -3680,8 +3769,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3713,6 +3804,7 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; + stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -3720,14 +3812,15 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; + /* Skip 1 cache entry */ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *fget = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -3752,9 +3845,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3774,9 +3869,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } @@ -3784,9 +3880,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3827,9 +3925,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); DISPATCH(); } @@ -3844,8 +3943,8 @@ "__build_class__ not found"); if (true) goto error; } - STACK_GROW(1); - stack_pointer[-1] = bc; + stack_pointer[0] = bc; + stack_pointer += 1; DISPATCH(); } @@ -3856,8 +3955,8 @@ PyObject *value; value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3873,8 +3972,8 @@ if (true) goto error; } Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3886,8 +3985,8 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3899,8 +3998,8 @@ value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3912,8 +4011,8 @@ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3929,9 +4028,9 @@ value2 = GETLOCAL(oparg2); Py_INCREF(value1); Py_INCREF(value2); - STACK_GROW(2); - stack_pointer[-2] = value1; - stack_pointer[-1] = value2; + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; DISPATCH(); } @@ -3984,8 +4083,8 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } @@ -4001,7 +4100,6 @@ INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; // _SPECIALIZE_LOAD_GLOBAL @@ -4019,6 +4117,9 @@ DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ // _LOAD_GLOBAL { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -4026,14 +4127,14 @@ && PyDict_CheckExact(BUILTINS())) { res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); + (PyDictObject *)BUILTINS(), + name); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } if (true) goto error; } @@ -4048,18 +4149,17 @@ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; if (res == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); if (true) goto error; } } } null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); DISPATCH(); } @@ -4067,8 +4167,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); @@ -4096,10 +4198,9 @@ STAT_INC(LOAD_GLOBAL, hit); null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); DISPATCH(); } @@ -4107,8 +4208,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); @@ -4117,6 +4220,7 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + /* Skip 1 cache entry */ // _LOAD_GLOBAL_MODULE { uint16_t index = read_u16(&this_instr[4].cache); @@ -4128,10 +4232,9 @@ STAT_INC(LOAD_GLOBAL, hit); null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); DISPATCH(); } @@ -4147,8 +4250,8 @@ if (true) goto error; } Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; + stack_pointer[0] = locals; + stack_pointer += 1; DISPATCH(); } @@ -4177,14 +4280,14 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } } - STACK_GROW(1); - stack_pointer[-1] = v; + stack_pointer[0] = v; + stack_pointer += 1; DISPATCH(); } @@ -4194,7 +4297,6 @@ INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *class; PyObject *global_super; PyObject *self; @@ -4224,8 +4326,8 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); if (err) goto pop_3_error; } // we make no attempt to optimize here; specializations should @@ -4258,10 +4360,9 @@ if (attr == NULL) goto pop_3_error; null = NULL; } - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-3] = attr; + if (oparg & 1) stack_pointer[-2] = null; + stack_pointer += -2 + (oparg & 1); DISPATCH(); } @@ -4269,10 +4370,12 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; PyObject *global_super; PyObject *attr; + /* Skip 1 cache entry */ self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -4286,8 +4389,8 @@ Py_DECREF(class); Py_DECREF(self); if (attr == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = attr; + stack_pointer[-3] = attr; + stack_pointer += -2 + ((0) ? 1 : 0); DISPATCH(); } @@ -4295,11 +4398,13 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; PyObject *global_super; PyObject *attr; PyObject *self_or_null; + /* Skip 1 cache entry */ self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -4324,9 +4429,9 @@ Py_DECREF(self); self_or_null = NULL; } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; DISPATCH(); } @@ -4352,17 +4457,14 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - + PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { GOTO_ERROR(error); } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; DISPATCH(); @@ -4382,7 +4484,7 @@ /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -4409,10 +4511,11 @@ } else { if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! attrs = Py_None; // Failure! } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + stack_pointer[-3] = attrs; + stack_pointer += -2; DISPATCH(); } @@ -4428,8 +4531,8 @@ // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + stack_pointer[0] = values_or_none; + stack_pointer += 1; DISPATCH(); } @@ -4442,8 +4545,8 @@ subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -4456,8 +4559,8 @@ subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -4475,8 +4578,8 @@ PyObject *exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - STACK_SHRINK(1); + Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + stack_pointer += -1; DISPATCH(); } @@ -4485,6 +4588,7 @@ next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyObject *cond; + /* Skip 1 cache entry */ cond = stack_pointer[-1]; assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); @@ -4492,7 +4596,7 @@ this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4503,6 +4607,7 @@ PyObject *value; PyObject *b; PyObject *cond; + /* Skip 1 cache entry */ // _IS_NONE value = stack_pointer[-1]; { @@ -4524,7 +4629,7 @@ #endif JUMPBY(oparg * flag); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4535,6 +4640,7 @@ PyObject *value; PyObject *b; PyObject *cond; + /* Skip 1 cache entry */ // _IS_NONE value = stack_pointer[-1]; { @@ -4556,7 +4662,7 @@ #endif JUMPBY(oparg * flag); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4565,6 +4671,7 @@ next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyObject *cond; + /* Skip 1 cache entry */ cond = stack_pointer[-1]; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); @@ -4572,7 +4679,7 @@ this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4583,7 +4690,7 @@ PyObject *value; value = stack_pointer[-1]; Py_DECREF(value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4603,9 +4710,9 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; DISPATCH(); } @@ -4615,8 +4722,8 @@ INSTRUCTION_STATS(PUSH_NULL); PyObject *res; res = NULL; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -4625,29 +4732,29 @@ next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; TIER_ONE_ONLY PyObject *cause = NULL, *exc = NULL; switch (oparg) { - case 2: + case 2: cause = args[1]; /* fall through */ - case 1: + case 1: exc = args[0]; /* fall through */ - case 0: + case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } break; - default: + default: _PyErr_SetString(tstate, PyExc_SystemError, "bad RAISE_VARARGS oparg"); break; } - if (true) { STACK_SHRINK(oparg); goto error; } + if (true) { stack_pointer += -oparg; goto error; } } TARGET(RERAISE) { @@ -4657,7 +4764,7 @@ PyObject *exc; PyObject **values; exc = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; TIER_ONE_ONLY assert(oparg >= 0 && oparg <= 2); if (oparg) { @@ -4683,6 +4790,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); + TIER_ONE_ONLY assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } @@ -4693,12 +4801,11 @@ INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); _Py_CODEUNIT *this_instr = next_instr - 1; - static_assert(0 == 0, "incorrect cache size"); TIER_ONE_ONLY assert(frame == tstate->current_frame); uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((code_version & 255) == 0); if (code_version != global_version) { @@ -4719,10 +4826,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); -#if defined(__EMSCRIPTEN__) + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; -#endif + #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); @@ -4736,7 +4844,7 @@ INSTRUCTION_STATS(RETURN_CONST); PyObject *value; PyObject *retval; - // LOAD_CONST + // _LOAD_CONST { value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); @@ -4744,11 +4852,11 @@ // _POP_FRAME retval = value; { - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -4757,12 +4865,12 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif + #endif } DISPATCH(); } @@ -4771,6 +4879,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); + TIER_ONE_ONLY assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4801,12 +4910,12 @@ INSTRUCTION_STATS(RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; - STACK_SHRINK(1); - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -4815,12 +4924,12 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif + #endif DISPATCH(); } @@ -4830,7 +4939,6 @@ INSTRUCTION_STATS(SEND); PREDICTED(SEND); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *receiver; PyObject *v; PyObject *retval; @@ -4897,8 +5005,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *v; PyObject *receiver; + /* Skip 1 cache entry */ v = stack_pointer[-1]; receiver = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, SEND); @@ -4955,7 +5065,7 @@ int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4971,28 +5081,28 @@ PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; default: - Py_UNREACHABLE(); + Py_UNREACHABLE(); } - STACK_SHRINK(1); - stack_pointer[-1] = func; + stack_pointer[-2] = func; + stack_pointer += -1; DISPATCH(); } @@ -5007,7 +5117,7 @@ int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5017,7 +5127,6 @@ INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; // _SPECIALIZE_STORE_ATTR @@ -5036,6 +5145,7 @@ DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } + /* Skip 3 cache entries */ // _STORE_ATTR v = stack_pointer[-2]; { @@ -5045,7 +5155,7 @@ Py_DECREF(owner); if (err) goto pop_2_error; } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5053,8 +5163,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -5086,7 +5198,7 @@ } Py_DECREF(owner); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5094,8 +5206,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -5115,7 +5229,7 @@ Py_XDECREF(old_value); Py_DECREF(owner); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5123,8 +5237,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; + /* Skip 1 cache entry */ owner = stack_pointer[-1]; value = stack_pointer[-2]; uint32_t type_version = read_u32(&this_instr[2].cache); @@ -5167,7 +5283,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5181,7 +5297,7 @@ PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5192,7 +5308,7 @@ PyObject *value; value = stack_pointer[-1]; SETLOCAL(oparg, value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5224,7 +5340,7 @@ uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); SETLOCAL(oparg2, value2); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5238,7 +5354,7 @@ int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5258,12 +5374,12 @@ if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, v); else - err = PyObject_SetItem(ns, name, v); + err = PyObject_SetItem(ns, name, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5291,7 +5407,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - STACK_SHRINK(4); + stack_pointer += -4; DISPATCH(); } @@ -5301,7 +5417,6 @@ INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *v; @@ -5331,7 +5446,7 @@ Py_DECREF(sub); if (err) goto pop_3_error; } - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } @@ -5339,9 +5454,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *dict; PyObject *value; + /* Skip 1 cache entry */ sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; @@ -5350,7 +5467,7 @@ int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } @@ -5358,29 +5475,29 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *list; PyObject *value; + /* Skip 1 cache entry */ sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } @@ -5404,7 +5521,6 @@ INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; // _SPECIALIZE_TO_BOOL @@ -5422,6 +5538,7 @@ DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } + /* Skip 2 cache entries */ // _TO_BOOL { int err = PyObject_IsTrue(value); @@ -5437,8 +5554,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; + /* Skip 1 cache entry */ value = stack_pointer[-1]; uint32_t version = read_u32(&this_instr[2].cache); // This one is a bit weird, because we expect *some* failures: @@ -5455,7 +5574,10 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -5466,8 +5588,11 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -5487,8 +5612,11 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -5502,8 +5630,11 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: DEOPT_IF(!Py_IsNone(value), TO_BOOL); @@ -5517,8 +5648,11 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -5587,7 +5721,7 @@ int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); if (res == 0) goto pop_1_error; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + stack_pointer += (oparg >> 8) + (oparg & 0xFF); DISPATCH(); } @@ -5597,7 +5731,6 @@ INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; // _SPECIALIZE_UNPACK_SEQUENCE seq = stack_pointer[-1]; @@ -5623,8 +5756,7 @@ Py_DECREF(seq); if (res == 0) goto pop_1_error; } - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5632,10 +5764,12 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; + /* Skip 1 cache entry */ seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5644,8 +5778,7 @@ *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5653,10 +5786,12 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; + /* Skip 1 cache entry */ seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5665,8 +5800,7 @@ *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5674,10 +5808,12 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; + /* Skip 1 cache entry */ seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -5685,8 +5821,7 @@ values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5708,9 +5843,8 @@ - exit_func: FOURTH = the context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */ + */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); tb = PyException_GetTraceback(val); @@ -5724,10 +5858,10 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -5737,6 +5871,7 @@ INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; + TIER_ONE_ONLY // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -5759,5 +5894,4 @@ LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } - #undef TIER_ONE diff --git a/Python/getargs.c b/Python/getargs.c index c0c2eb27184e3c..0c4ce282f48764 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1,6 +1,7 @@ /* New getargs implementation */ +#define PY_CXX_CONST const #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_dict.h" // _PyDict_HasOnlyStringKeys() @@ -12,10 +13,10 @@ PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, const char *, ...); PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywords_SizeT(PyObject *, PyObject *, - const char *, char **, ...); + const char *, const char * const *, ...); PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, const char *, va_list); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *, PyObject *, - const char *, char **, va_list); + const char *, const char * const *, va_list); #define FLAG_COMPAT 1 @@ -54,7 +55,7 @@ static Py_ssize_t convertbuffer(PyObject *, const void **p, const char **); static int getbuffer(PyObject *, Py_buffer *, const char**); static int vgetargskeywords(PyObject *, PyObject *, - const char *, char **, va_list *, int); + const char *, const char * const *, va_list *, int); static int vgetargskeywordsfast(PyObject *, PyObject *, struct _PyArg_Parser *, va_list *, int); static int vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, @@ -477,7 +478,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else if (c == ':' || c == ';' || c == '\0') break; - else if (level == 0 && Py_ISALPHA(c)) + else if (level == 0 && Py_ISALPHA(c) && c != 'e') n++; } @@ -1247,7 +1248,7 @@ int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *keywords, const char *format, - char **kwlist, ...) + const char * const *kwlist, ...) { int retval; va_list va; @@ -1271,7 +1272,7 @@ int _PyArg_ParseTupleAndKeywords_SizeT(PyObject *args, PyObject *keywords, const char *format, - char **kwlist, ...) + const char * const *kwlist, ...) { int retval; va_list va; @@ -1297,7 +1298,7 @@ int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *keywords, const char *format, - char **kwlist, va_list va) + const char * const *kwlist, va_list va) { int retval; va_list lva; @@ -1322,7 +1323,7 @@ int _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *args, PyObject *keywords, const char *format, - char **kwlist, va_list va) + const char * const *kwlist, va_list va) { int retval; va_list lva; @@ -1460,7 +1461,7 @@ PyArg_ValidateKeywordArguments(PyObject *kwargs) static int vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, - char **kwlist, va_list *p_va, int flags) + const char * const *kwlist, va_list *p_va, int flags) { char msgbuf[512]; int levels[32]; diff --git a/Python/hashtable.c b/Python/hashtable.c index 8f5e8168ba1339..faf68fe4ff0bca 100644 --- a/Python/hashtable.c +++ b/Python/hashtable.c @@ -45,7 +45,7 @@ */ #include "Python.h" -#include "pycore_hashtable.h" +#include "pycore_hashtable.h" // export _Py_hashtable_new() #include "pycore_pyhash.h" // _Py_HashPointerRaw() #define HASHTABLE_MIN_SIZE 16 diff --git a/Python/import.c b/Python/import.c index f37393bbdc4269..2dd95d8364a0be 100644 --- a/Python/import.c +++ b/Python/import.c @@ -252,18 +252,21 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n NOTE: because of this, initializing must be set *before* stuffing the new module in sys.modules. */ - spec = PyObject_GetAttr(mod, &_Py_ID(__spec__)); - int busy = _PyModuleSpec_IsInitializing(spec); - Py_XDECREF(spec); - if (busy) { - /* Wait until module is done importing. */ - PyObject *value = PyObject_CallMethodOneArg( - IMPORTLIB(interp), &_Py_ID(_lock_unlock_module), name); - if (value == NULL) { - return -1; - } - Py_DECREF(value); + int rc = PyObject_GetOptionalAttr(mod, &_Py_ID(__spec__), &spec); + if (rc > 0) { + rc = _PyModuleSpec_IsInitializing(spec); + Py_DECREF(spec); + } + if (rc <= 0) { + return rc; + } + /* Wait until module is done importing. */ + PyObject *value = PyObject_CallMethodOneArg( + IMPORTLIB(interp), &_Py_ID(_lock_unlock_module), name); + if (value == NULL) { + return -1; } + Py_DECREF(value); return 0; } @@ -415,11 +418,7 @@ remove_module(PyThreadState *tstate, PyObject *name) Py_ssize_t _PyImport_GetNextModuleIndex(void) { - PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); - LAST_MODULE_INDEX++; - Py_ssize_t index = LAST_MODULE_INDEX; - PyThread_release_lock(EXTENSIONS.mutex); - return index; + return _Py_atomic_add_ssize(&LAST_MODULE_INDEX, 1) + 1; } static const char * @@ -879,13 +878,13 @@ gets even messier. static inline void extensions_lock_acquire(void) { - PyThread_acquire_lock(_PyRuntime.imports.extensions.mutex, WAIT_LOCK); + PyMutex_Lock(&_PyRuntime.imports.extensions.mutex); } static inline void extensions_lock_release(void) { - PyThread_release_lock(_PyRuntime.imports.extensions.mutex); + PyMutex_Unlock(&_PyRuntime.imports.extensions.mutex); } /* Magic for extension modules (built-in as well as dynamically diff --git a/Python/initconfig.c b/Python/initconfig.c index d7f3195ed5fcf0..06e317907b8ec9 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -293,6 +293,8 @@ static const char usage_envvars[] = "PYTHON_FROZEN_MODULES : if this variable is set, it determines whether or not \n" " frozen modules should be used. The default is \"on\" (or \"off\" if you are \n" " running a local build).\n" +"PYTHON_COLORS : If this variable is set to 1, the interpreter will" +" colorize various kinds of output. Setting it to 0 deactivates this behavior.\n" "These variables have equivalent command-line parameters (see --help for details):\n" "PYTHONDEBUG : enable parser debug mode (-d)\n" "PYTHONDONTWRITEBYTECODE : don't write .pyc files (-B)\n" diff --git a/Python/lock.c b/Python/lock.c index e9279f0b92a5e7..f0ff1176941da8 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -353,3 +353,109 @@ _PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) v = _Py_atomic_load_uint8(&flag->v); } } + +#define _Py_WRITE_LOCKED 1 +#define _PyRWMutex_READER_SHIFT 2 +#define _Py_RWMUTEX_MAX_READERS (UINTPTR_MAX >> _PyRWMutex_READER_SHIFT) + +static uintptr_t +rwmutex_set_parked_and_wait(_PyRWMutex *rwmutex, uintptr_t bits) +{ + // Set _Py_HAS_PARKED and wait until we are woken up. + if ((bits & _Py_HAS_PARKED) == 0) { + uintptr_t newval = bits | _Py_HAS_PARKED; + if (!_Py_atomic_compare_exchange_uintptr(&rwmutex->bits, + &bits, newval)) { + return bits; + } + bits = newval; + } + + _PyParkingLot_Park(&rwmutex->bits, &bits, sizeof(bits), -1, NULL, 1); + return _Py_atomic_load_uintptr_relaxed(&rwmutex->bits); +} + +// The number of readers holding the lock +static uintptr_t +rwmutex_reader_count(uintptr_t bits) +{ + return bits >> _PyRWMutex_READER_SHIFT; +} + +void +_PyRWMutex_RLock(_PyRWMutex *rwmutex) +{ + uintptr_t bits = _Py_atomic_load_uintptr_relaxed(&rwmutex->bits); + for (;;) { + if ((bits & _Py_WRITE_LOCKED)) { + // A writer already holds the lock. + bits = rwmutex_set_parked_and_wait(rwmutex, bits); + continue; + } + else if ((bits & _Py_HAS_PARKED)) { + // Reader(s) hold the lock (or just gave up the lock), but there is + // at least one waiting writer. We can't grab the lock because we + // don't want to starve the writer. Instead, we park ourselves and + // wait for the writer to eventually wake us up. + bits = rwmutex_set_parked_and_wait(rwmutex, bits); + continue; + } + else { + // The lock is unlocked or read-locked. Try to grab it. + assert(rwmutex_reader_count(bits) < _Py_RWMUTEX_MAX_READERS); + uintptr_t newval = bits + (1 << _PyRWMutex_READER_SHIFT); + if (!_Py_atomic_compare_exchange_uintptr(&rwmutex->bits, + &bits, newval)) { + continue; + } + return; + } + } +} + +void +_PyRWMutex_RUnlock(_PyRWMutex *rwmutex) +{ + uintptr_t bits = _Py_atomic_add_uintptr(&rwmutex->bits, -(1 << _PyRWMutex_READER_SHIFT)); + assert(rwmutex_reader_count(bits) > 0 && "lock was not read-locked"); + bits -= (1 << _PyRWMutex_READER_SHIFT); + + if (rwmutex_reader_count(bits) == 0 && (bits & _Py_HAS_PARKED)) { + _PyParkingLot_UnparkAll(&rwmutex->bits); + return; + } +} + +void +_PyRWMutex_Lock(_PyRWMutex *rwmutex) +{ + uintptr_t bits = _Py_atomic_load_uintptr_relaxed(&rwmutex->bits); + for (;;) { + // If there are no active readers and it's not already write-locked, + // then we can grab the lock. + if ((bits & ~_Py_HAS_PARKED) == 0) { + if (!_Py_atomic_compare_exchange_uintptr(&rwmutex->bits, + &bits, + bits | _Py_WRITE_LOCKED)) { + continue; + } + return; + } + + // Otherwise, we have to wait. + bits = rwmutex_set_parked_and_wait(rwmutex, bits); + } +} + +void +_PyRWMutex_Unlock(_PyRWMutex *rwmutex) +{ + uintptr_t old_bits = _Py_atomic_exchange_uintptr(&rwmutex->bits, 0); + + assert((old_bits & _Py_WRITE_LOCKED) && "lock was not write-locked"); + assert(rwmutex_reader_count(old_bits) == 0 && "lock was read-locked"); + + if ((old_bits & _Py_HAS_PARKED) != 0) { + _PyParkingLot_UnparkAll(&rwmutex->bits); + } +} diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index bcd6ea7564f9b3..e664e638bdb749 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -254,4 +254,5 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_LINE, - &&_unknown_opcode}; + &&_unknown_opcode, +}; diff --git a/Python/optimizer.c b/Python/optimizer.c index ec59fea26fbc70..f27af14d967cd3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -6,14 +6,20 @@ #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE #include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_uop_ids.h" #include "pycore_uops.h" #include "cpython/optimizer.h" #include #include #include +#define NEED_OPCODE_METADATA +#include "pycore_uop_metadata.h" // Uop tables +#undef NEED_OPCODE_METADATA + #define MAX_EXECUTORS_SIZE 256 + static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { @@ -167,6 +173,7 @@ _PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNI } _PyOptimizerObject *opt = interp->optimizer; _PyExecutorObject *executor = NULL; + /* Start optimizing at the destination to guarantee forward progress */ int err = opt->optimize(opt, code, dest, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { assert(executor == NULL); @@ -247,14 +254,13 @@ PyTypeObject _PyCounterExecutor_Type = { .tp_methods = executor_methods, }; -static _PyInterpreterFrame * +static _Py_CODEUNIT * counter_execute(_PyExecutorObject *self, _PyInterpreterFrame *frame, PyObject **stack_pointer) { ((_PyCounterExecutorObject *)self)->optimizer->count++; _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = ((_PyCounterExecutorObject *)self)->next_instr; Py_DECREF(self); - return frame; + return ((_PyCounterExecutorObject *)self)->next_instr; } static int @@ -325,11 +331,8 @@ uop_dealloc(_PyUOpExecutorObject *self) { } const char * -_PyUopName(int index) +_PyUOpName(int index) { - if (index <= MAX_REAL_OPCODE) { - return _PyOpcode_OpName[index]; - } return _PyOpcode_uop_name[index]; } @@ -347,7 +350,7 @@ uop_item(_PyUOpExecutorObject *self, Py_ssize_t index) PyErr_SetNone(PyExc_IndexError); return NULL; } - const char *name = _PyUopName(self->trace[index].opcode); + const char *name = _PyUOpName(self->trace[index].opcode); if (name == NULL) { name = ""; } @@ -388,7 +391,7 @@ PyTypeObject _PyUOpExecutor_Type = { /* TO DO -- Generate these tables */ static const uint16_t -_PyUop_Replacements[OPCODE_METADATA_SIZE] = { +_PyUOp_Replacements[MAX_UOP_ID + 1] = { [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE, [_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST, [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, @@ -409,6 +412,9 @@ BRANCH_TO_GUARD[4][2] = { #define TRACE_STACK_SIZE 5 +#define CONFIDENCE_RANGE 1000 +#define CONFIDENCE_CUTOFF 333 + /* Returns 1 on success, * 0 if it failed to produce a worthwhile trace, * and -1 on an error. @@ -431,6 +437,7 @@ translate_bytecode_to_trace( _Py_CODEUNIT *instr; } trace_stack[TRACE_STACK_SIZE]; int trace_stack_depth = 0; + int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -451,7 +458,7 @@ translate_bytecode_to_trace( #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ DPRINTF(2, \ " ADD_TO_TRACE(%s, %d, %" PRIu64 ")\n", \ - _PyUopName(OPCODE), \ + _PyUOpName(OPCODE), \ (OPARG), \ (uint64_t)(OPERAND)); \ assert(trace_length < max_length); \ @@ -474,7 +481,7 @@ translate_bytecode_to_trace( } // Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE -#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUopName(opcode)) +#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) // Trace stack operations (used by _PUSH_FRAME, _POP_FRAME) #define TRACE_STACK_PUSH() \ @@ -511,12 +518,11 @@ translate_bytecode_to_trace( uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; - uint32_t extras = 0; - + uint32_t extended = 0; if (opcode == EXTENDED_ARG) { instr++; - extras += 1; + extended = 1; opcode = instr->op.code; oparg = (oparg << 8) | instr->op.arg; if (opcode == EXTENDED_ARG) { @@ -543,11 +549,22 @@ translate_bytecode_to_trace( int counter = instr[1].cache; int bitcount = _Py_popcount32(counter); int jump_likely = bitcount > 8; + if (jump_likely) { + confidence = confidence * bitcount / 16; + } + else { + confidence = confidence * (16 - bitcount) / 16; + } + if (confidence < CONFIDENCE_CUTOFF) { + DPRINTF(2, "Confidence too low (%d)\n", confidence); + OPT_STAT_INC(low_confidence); + goto done; + } uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, uopcode=%s\n", - _PyUopName(opcode), oparg, - counter, bitcount, jump_likely, _PyUopName(uopcode)); + DPRINTF(2, "%s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", + _PyUOpName(opcode), oparg, + counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); ADD_TO_TRACE(uopcode, max_length, 0, target); if (jump_likely) { _Py_CODEUNIT *target_instr = next_instr + oparg; @@ -560,6 +577,7 @@ translate_bytecode_to_trace( } case JUMP_BACKWARD: + case JUMP_BACKWARD_NO_INTERRUPT: { if (instr + 2 - oparg == initial_instr && code == initial_code) { RESERVE(1); @@ -606,23 +624,7 @@ translate_bytecode_to_trace( int offset = expansion->uops[i].offset + 1; switch (expansion->uops[i].size) { case OPARG_FULL: - if (extras && OPCODE_HAS_JUMP(opcode)) { - if (opcode == JUMP_BACKWARD_NO_INTERRUPT) { - oparg -= extras; - } - else { - assert(opcode != JUMP_BACKWARD); - oparg += extras; - } - } - if (_PyUop_Replacements[uop]) { - uop = _PyUop_Replacements[uop]; - if (uop == _FOR_ITER_TIER_TWO) { - target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; - assert(_PyCode_CODE(code)[target-1].op.code == END_FOR || - _PyCode_CODE(code)[target-1].op.code == INSTRUMENTED_END_FOR); - } - } + assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); break; case OPARG_CACHE_1: operand = read_u16(&instr[offset].cache); @@ -643,7 +645,15 @@ translate_bytecode_to_trace( oparg = offset; assert(uop == _SAVE_RETURN_OFFSET); break; - + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); + if (uop == _FOR_ITER_TIER_TWO) { + target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1 + extended; + assert(_PyCode_CODE(code)[target-1].op.code == END_FOR || + _PyCode_CODE(code)[target-1].op.code == INSTRUMENTED_END_FOR); + } + break; default: fprintf(stderr, "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", @@ -712,7 +722,7 @@ translate_bytecode_to_trace( } break; } - DPRINTF(2, "Unsupported opcode %s\n", _PyUopName(opcode)); + DPRINTF(2, "Unsupported opcode %s\n", _PyUOpName(opcode)); OPT_UNSUPPORTED_OPCODE(opcode); goto done; // Break out of loop } // End default @@ -785,7 +795,8 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) } /* All other micro-ops fall through, so i+1 is reachable */ SET_BIT(used, i+1); - if (OPCODE_HAS_JUMP(opcode)) { + assert(opcode <= MAX_UOP_ID); + if (_PyUop_Flags[opcode] & HAS_JUMP_FLAG) { /* Mark target as reachable */ SET_BIT(used, buffer[i].oparg); } @@ -832,7 +843,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) dest--; } assert(dest == -1); - executor->base.execute = _PyUopExecute; + executor->base.execute = _PyUOpExecute; _Py_ExecutorInit((_PyExecutorObject *)executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -845,7 +856,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) for (int i = 0; i < length; i++) { printf("%4d %s(%d, %d, %" PRIu64 ")\n", i, - _PyUopName(executor->trace[i].opcode), + _PyUOpName(executor->trace[i].opcode), executor->trace[i].oparg, executor->trace[i].target, executor->trace[i].operand); @@ -888,11 +899,11 @@ uop_optimize( return 1; } -/* Dummy execute() function for Uop Executor. +/* Dummy execute() function for UOp Executor. * The actual implementation is inlined in ceval.c, * in _PyEval_EvalFrameDefault(). */ -_PyInterpreterFrame * -_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) +_Py_CODEUNIT * +_PyUOpExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) { Py_FatalError("Tier 2 is now inlined into Tier 1"); } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0f9bc085f22f1c..4eb2d9711f5e56 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -4,6 +4,7 @@ #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_uop_metadata.h" #include "pycore_uops.h" #include "pycore_long.h" #include "cpython/optimizer.h" @@ -15,7 +16,6 @@ static void remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { - // Note that we don't enter stubs, those SET_IPs are needed. int last_set_ip = -1; bool maybe_invalid = false; for (int pc = 0; pc < buffer_size; pc++) { @@ -36,13 +36,13 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) break; } else { - if (OPCODE_HAS_ESCAPES(opcode)) { + if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { maybe_invalid = true; if (last_set_ip >= 0) { buffer[last_set_ip].opcode = _SET_IP; } } - if (OPCODE_HAS_ERROR(opcode) || opcode == _PUSH_FRAME) { + if ((_PyUop_Flags[opcode] & HAS_ERROR_FLAG) || opcode == _PUSH_FRAME) { if (last_set_ip >= 0) { buffer[last_set_ip].opcode = _SET_IP; } diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 664e622cc17474..d44c1b4b93b4d2 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -118,10 +118,19 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) if (timeout >= 0) { struct timespec ts; +#if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) + _PyTime_t deadline = _PyTime_Add(_PyTime_GetMonotonicClock(), timeout); + + _PyTime_AsTimespec_clamp(deadline, &ts); + + err = sem_clockwait(&sema->platform_sem, CLOCK_MONOTONIC, &ts); +#else _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); - _PyTime_AsTimespec(deadline, &ts); + + _PyTime_AsTimespec_clamp(deadline, &ts); err = sem_timedwait(&sema->platform_sem, &ts); +#endif } else { err = sem_wait(&sema->platform_sem); @@ -151,7 +160,7 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) struct timespec ts; _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); - _PyTime_AsTimespec(deadline, &ts); + _PyTime_AsTimespec_clamp(deadline, &ts); err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); } diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 208ced6c101dce..750ba18d3510ed 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -216,10 +216,24 @@ perf_map_write_entry(void *state, const void *code_addr, PyMem_RawFree(perf_map_entry); } +static void* +perf_map_init_state(void) +{ + PyUnstable_PerfMapState_Init(); + return NULL; +} + +static int +perf_map_free_state(void *state) +{ + PyUnstable_PerfMapState_Fini(); + return 0; +} + _PyPerf_Callbacks _Py_perfmap_callbacks = { - NULL, + &perf_map_init_state, &perf_map_write_entry, - NULL, + &perf_map_free_state, }; static int @@ -233,7 +247,7 @@ new_code_arena(void) mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, // fd (not used here) 0); // offset (not used here) - if (!memory) { + if (memory == MAP_FAILED) { PyErr_SetFromErrno(PyExc_OSError); PyErr_FormatUnraisable("Failed to create new mmap for perf trampoline"); perf_status = PERF_STATUS_FAILED; @@ -415,7 +429,6 @@ _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *callbacks) trampoline_api.write_state = callbacks->write_state; trampoline_api.free_state = callbacks->free_state; trampoline_api.state = NULL; - perf_status = PERF_STATUS_OK; #endif return 0; } @@ -434,6 +447,7 @@ _PyPerfTrampoline_Init(int activate) } if (!activate) { tstate->interp->eval_frame = NULL; + perf_status = PERF_STATUS_NO_INIT; } else { tstate->interp->eval_frame = py_trampoline_evaluator; @@ -444,6 +458,9 @@ _PyPerfTrampoline_Init(int activate) if (extra_code_index == -1) { return -1; } + if (trampoline_api.state == NULL && trampoline_api.init_state != NULL) { + trampoline_api.state = trampoline_api.init_state(); + } perf_status = PERF_STATUS_OK; } #endif @@ -454,16 +471,29 @@ int _PyPerfTrampoline_Fini(void) { #ifdef PY_HAVE_PERF_TRAMPOLINE + if (perf_status != PERF_STATUS_OK) { + return 0; + } PyThreadState *tstate = _PyThreadState_GET(); if (tstate->interp->eval_frame == py_trampoline_evaluator) { tstate->interp->eval_frame = NULL; } - free_code_arenas(); + if (perf_status == PERF_STATUS_OK) { + trampoline_api.free_state(trampoline_api.state); + } extra_code_index = -1; + perf_status = PERF_STATUS_NO_INIT; #endif return 0; } +void _PyPerfTrampoline_FreeArenas(void) { +#ifdef PY_HAVE_PERF_TRAMPOLINE + free_code_arenas(); +#endif + return; +} + int PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable){ #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -477,8 +507,8 @@ PyStatus _PyPerfTrampoline_AfterFork_Child(void) { #ifdef PY_HAVE_PERF_TRAMPOLINE - PyUnstable_PerfMapState_Fini(); if (persist_after_fork) { + _PyPerfTrampoline_Fini(); char filename[256]; pid_t parent_pid = getppid(); snprintf(filename, sizeof(filename), "/tmp/perf-%d.map", parent_pid); diff --git a/Python/pyhash.c b/Python/pyhash.c index f9060b8003a0a7..141407c265677a 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -83,8 +83,6 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; */ -Py_hash_t _Py_HashPointer(const void *); - Py_hash_t _Py_HashDouble(PyObject *inst, double v) { @@ -132,23 +130,13 @@ _Py_HashDouble(PyObject *inst, double v) } Py_hash_t -_Py_HashPointerRaw(const void *p) -{ - size_t y = (size_t)p; - /* bottom 3 or 4 bits are likely to be 0; rotate y by 4 to avoid - excessive hash collisions for dicts and sets */ - y = (y >> 4) | (y << (8 * SIZEOF_VOID_P - 4)); - return (Py_hash_t)y; -} - -Py_hash_t -_Py_HashPointer(const void *p) +Py_HashPointer(const void *ptr) { - Py_hash_t x = _Py_HashPointerRaw(p); - if (x == -1) { - x = -2; + Py_hash_t hash = _Py_HashPointerRaw(ptr); + if (hash == -1) { + hash = -2; } - return x; + return hash; } Py_hash_t diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ac8d5208322882..1d8af26e4a1cb7 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -528,11 +528,6 @@ pycore_init_runtime(_PyRuntimeState *runtime, return status; } - status = _PyTime_Init(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - status = _PyImport_Init(); if (_PyStatus_EXCEPTION(status)) { return status; @@ -581,44 +576,33 @@ init_interp_settings(PyInterpreterState *interp, interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS; } - /* We check "gil" in init_interp_create_gil(). */ + switch (config->gil) { + case PyInterpreterConfig_DEFAULT_GIL: break; + case PyInterpreterConfig_SHARED_GIL: break; + case PyInterpreterConfig_OWN_GIL: break; + default: + return _PyStatus_ERR("invalid interpreter config 'gil' value"); + } return _PyStatus_OK(); } -static PyStatus +static void init_interp_create_gil(PyThreadState *tstate, int gil) { - PyStatus status; - /* finalize_interp_delete() comment explains why _PyEval_FiniGIL() is only called here. */ // XXX This is broken with a per-interpreter GIL. _PyEval_FiniGIL(tstate->interp); /* Auto-thread-state API */ - status = _PyGILState_SetTstate(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + _PyGILState_SetTstate(tstate); - int own_gil; - switch (gil) { - case PyInterpreterConfig_DEFAULT_GIL: own_gil = 0; break; - case PyInterpreterConfig_SHARED_GIL: own_gil = 0; break; - case PyInterpreterConfig_OWN_GIL: own_gil = 1; break; - default: - return _PyStatus_ERR("invalid interpreter config 'gil' value"); - } + int own_gil = (gil == PyInterpreterConfig_OWN_GIL); /* Create the GIL and take it */ - status = _PyEval_InitGIL(tstate, own_gil); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - return _PyStatus_OK(); + _PyEval_InitGIL(tstate, own_gil); } @@ -662,10 +646,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, } _PyThreadState_Bind(tstate); - status = init_interp_create_gil(tstate, config.gil); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + init_interp_create_gil(tstate, config.gil); *tstate_p = tstate; return _PyStatus_OK(); @@ -739,6 +720,11 @@ pycore_init_types(PyInterpreterState *interp) return status; } + status = _PyXI_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } @@ -825,6 +811,11 @@ pycore_interp_init(PyThreadState *tstate) return status; } + status = _PyDtoa_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + // The GC must be initialized before the first GC collection. status = _PyGC_Init(interp); if (_PyStatus_EXCEPTION(status)) { @@ -1742,6 +1733,7 @@ finalize_interp_types(PyInterpreterState *interp) { _PyUnicode_FiniTypes(interp); _PySys_FiniTypes(interp); + _PyXI_FiniTypes(interp); _PyExc_Fini(interp); _PyAsyncGen_Fini(interp); _PyContext_Fini(interp); @@ -1781,6 +1773,7 @@ finalize_interp_clear(PyThreadState *tstate) _PyXI_Fini(tstate->interp); _PyExc_ClearExceptionGroupType(tstate->interp); _Py_clear_generic_types(tstate->interp); + _PyDtoa_Fini(tstate->interp); /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); @@ -1797,9 +1790,14 @@ finalize_interp_clear(PyThreadState *tstate) _PyArg_Fini(); _Py_ClearFileSystemEncoding(); _PyPerfTrampoline_Fini(); + _PyPerfTrampoline_FreeArenas(); } finalize_interp_types(tstate->interp); + + /* finalize_interp_types may allocate Python objects so we may need to + abandon mimalloc segments again */ + _PyThreadState_ClearMimallocHeaps(tstate); } @@ -1854,7 +1852,6 @@ Py_FinalizeEx(void) */ _PyAtExit_Call(tstate->interp); - PyUnstable_PerfMapState_Fini(); /* Copy the core config, PyInterpreterState_Delete() free the core config memory */ @@ -2092,28 +2089,21 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) return _PyStatus_OK(); } - PyThreadState *tstate = _PyThreadState_New(interp, - _PyThreadState_WHENCE_INTERP); - if (tstate == NULL) { - PyInterpreterState_Delete(interp); - *tstate_p = NULL; - return _PyStatus_OK(); - } - _PyThreadState_Bind(tstate); - + // XXX Might new_interpreter() have been called without the GIL held? PyThreadState *save_tstate = _PyThreadState_GET(); - int has_gil = 0; + PyThreadState *tstate = NULL; /* From this point until the init_interp_create_gil() call, we must not do anything that requires that the GIL be held (or otherwise exist). That applies whether or not the new interpreter has its own GIL (e.g. the main interpreter). */ + if (save_tstate != NULL) { + _PyThreadState_Detach(save_tstate); + } /* Copy the current interpreter config into the new interpreter */ const PyConfig *src_config; if (save_tstate != NULL) { - // XXX Might new_interpreter() have been called without the GIL held? - _PyThreadState_Detach(save_tstate); src_config = _PyInterpreterState_GetConfig(save_tstate->interp); } else @@ -2135,11 +2125,14 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } - status = init_interp_create_gil(tstate, config->gil); - if (_PyStatus_EXCEPTION(status)) { + tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INTERP); + if (tstate == NULL) { + status = _PyStatus_NO_MEMORY(); goto error; } - has_gil = 1; + + _PyThreadState_Bind(tstate); + init_interp_create_gil(tstate, config->gil); /* No objects have been created yet. */ @@ -2158,16 +2151,14 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) error: *tstate_p = NULL; - - /* Oops, it didn't work. Undo it all. */ - if (has_gil) { + if (tstate != NULL) { + PyThreadState_Clear(tstate); _PyThreadState_Detach(tstate); + PyThreadState_Delete(tstate); } if (save_tstate != NULL) { _PyThreadState_Attach(save_tstate); } - PyThreadState_Clear(tstate); - PyThreadState_Delete(tstate); PyInterpreterState_Delete(interp); return status; @@ -3055,13 +3046,13 @@ wait_for_thread_shutdown(PyThreadState *tstate) int Py_AtExit(void (*func)(void)) { struct _atexit_runtime_state *state = &_PyRuntime.atexit; - PyThread_acquire_lock(state->mutex, WAIT_LOCK); + PyMutex_Lock(&state->mutex); if (state->ncallbacks >= NEXITFUNCS) { - PyThread_release_lock(state->mutex); + PyMutex_Unlock(&state->mutex); return -1; } state->callbacks[state->ncallbacks++] = func; - PyThread_release_lock(state->mutex); + PyMutex_Unlock(&state->mutex); return 0; } @@ -3071,18 +3062,18 @@ call_ll_exitfuncs(_PyRuntimeState *runtime) atexit_callbackfunc exitfunc; struct _atexit_runtime_state *state = &runtime->atexit; - PyThread_acquire_lock(state->mutex, WAIT_LOCK); + PyMutex_Lock(&state->mutex); while (state->ncallbacks > 0) { /* pop last function from the list */ state->ncallbacks--; exitfunc = state->callbacks[state->ncallbacks]; state->callbacks[state->ncallbacks] = NULL; - PyThread_release_lock(state->mutex); + PyMutex_Unlock(&state->mutex); exitfunc(); - PyThread_acquire_lock(state->mutex, WAIT_LOCK); + PyMutex_Lock(&state->mutex); } - PyThread_release_lock(state->mutex); + PyMutex_Unlock(&state->mutex); fflush(stdout); fflush(stderr); diff --git a/Python/pystate.c b/Python/pystate.c index 6196b15da0117a..84e2d6ea172f2b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -236,6 +236,8 @@ tstate_is_bound(PyThreadState *tstate) static void bind_gilstate_tstate(PyThreadState *); static void unbind_gilstate_tstate(PyThreadState *); +static void tstate_mimalloc_bind(PyThreadState *); + static void bind_tstate(PyThreadState *tstate) { @@ -256,6 +258,9 @@ bind_tstate(PyThreadState *tstate) tstate->native_thread_id = PyThread_get_thread_native_id(); #endif + // mimalloc state needs to be initialized from the active thread. + tstate_mimalloc_bind(tstate); + tstate->_status.bound = 1; } @@ -379,49 +384,23 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime); _Py_COMP_DIAG_POP -#define NUMLOCKS 8 #define LOCKS_INIT(runtime) \ { \ &(runtime)->interpreters.mutex, \ &(runtime)->xi.registry.mutex, \ - &(runtime)->unicode_state.ids.lock, \ + &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ - &(runtime)->ceval.pending_mainthread.lock, \ + &(runtime)->ceval.pending_mainthread.mutex, \ &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ } -static int -alloc_for_runtime(PyThread_type_lock locks[NUMLOCKS]) -{ - /* Force default allocator, since _PyRuntimeState_Fini() must - use the same allocator than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - for (int i = 0; i < NUMLOCKS; i++) { - PyThread_type_lock lock = PyThread_allocate_lock(); - if (lock == NULL) { - for (int j = 0; j < i; j++) { - PyThread_free_lock(locks[j]); - locks[j] = NULL; - } - break; - } - locks[i] = lock; - } - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return 0; -} - static void init_runtime(_PyRuntimeState *runtime, void *open_code_hook, void *open_code_userdata, _Py_AuditHookEntry *audit_hook_head, - Py_ssize_t unicode_next_index, - PyThread_type_lock locks[NUMLOCKS]) + Py_ssize_t unicode_next_index) { assert(!runtime->preinitializing); assert(!runtime->preinitialized); @@ -435,12 +414,6 @@ init_runtime(_PyRuntimeState *runtime, PyPreConfig_InitPythonConfig(&runtime->preconfig); - PyThread_type_lock *lockptrs[NUMLOCKS] = LOCKS_INIT(runtime); - for (int i = 0; i < NUMLOCKS; i++) { - assert(locks[i] != NULL); - *lockptrs[i] = locks[i]; - } - // Set it to the ID of the main thread of the main interpreter. runtime->main_thread = PyThread_get_thread_ident(); @@ -466,11 +439,6 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) // is called multiple times. Py_ssize_t unicode_next_index = runtime->unicode_state.ids.next_index; - PyThread_type_lock locks[NUMLOCKS]; - if (alloc_for_runtime(locks) != 0) { - return _PyStatus_NO_MEMORY(); - } - if (runtime->_initialized) { // Py_Initialize() must be running again. // Reset to _PyRuntimeState_INIT. @@ -489,7 +457,7 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) } init_runtime(runtime, open_code_hook, open_code_userdata, audit_hook_head, - unicode_next_index, locks); + unicode_next_index); return _PyStatus_OK(); } @@ -509,23 +477,6 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) if (PyThread_tss_is_created(&runtime->trashTSSkey)) { PyThread_tss_delete(&runtime->trashTSSkey); } - - /* Force the allocator used by _PyRuntimeState_Init(). */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -#define FREE_LOCK(LOCK) \ - if (LOCK != NULL) { \ - PyThread_free_lock(LOCK); \ - LOCK = NULL; \ - } - - PyThread_type_lock *lockptrs[NUMLOCKS] = LOCKS_INIT(runtime); - for (int i = 0; i < NUMLOCKS; i++) { - FREE_LOCK(*lockptrs[i]); - } - -#undef FREE_LOCK - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } #ifdef HAVE_FORK @@ -537,28 +488,19 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) // This was initially set in _PyRuntimeState_Init(). runtime->main_thread = PyThread_get_thread_ident(); - /* Force default allocator, since _PyRuntimeState_Fini() must - use the same allocator than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - PyThread_type_lock *lockptrs[NUMLOCKS] = LOCKS_INIT(runtime); - int reinit_err = 0; - for (int i = 0; i < NUMLOCKS; i++) { - reinit_err += _PyThread_at_fork_reinit(lockptrs[i]); - } - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - // Clears the parking lot. Any waiting threads are dead. This must be // called before releasing any locks that use the parking lot. _PyParkingLot_AfterFork(); + // Re-initialize global locks + PyMutex *locks[] = LOCKS_INIT(runtime); + for (size_t i = 0; i < Py_ARRAY_LENGTH(locks); i++) { + _PyMutex_at_fork_reinit(locks[i]); + } + /* bpo-42540: id_mutex is freed by _PyInterpreterState_Delete, which does * not force the default allocator. */ - reinit_err += _PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex); - - if (reinit_err < 0) { + if (_PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex) < 0) { return _PyStatus_ERR("Failed to reinitialize runtime locks"); } @@ -594,24 +536,6 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) { struct pyinterpreters *interpreters = &runtime->interpreters; interpreters->next_id = 0; - - /* Py_Finalize() calls _PyRuntimeState_Fini() which clears the mutex. - Create a new mutex if needed. */ - if (interpreters->mutex == NULL) { - /* Force default allocator, since _PyRuntimeState_Fini() must - use the same allocator than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - interpreters->mutex = PyThread_allocate_lock(); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - if (interpreters->mutex == NULL) { - return _PyStatus_ERR("Can't initialize threads for interpreter"); - } - } - return _PyStatus_OK(); } @@ -654,8 +578,7 @@ free_interpreter(PyInterpreterState *interp) static PyStatus init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, - PyInterpreterState *next, - PyThread_type_lock pending_lock) + PyInterpreterState *next) { if (interp->_initialized) { return _PyStatus_ERR("interpreter already initialized"); @@ -684,7 +607,7 @@ init_interpreter(PyInterpreterState *interp, return status; } - _PyEval_InitState(interp, pending_lock); + _PyEval_InitState(interp); _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); @@ -730,11 +653,6 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) } } - PyThread_type_lock pending_lock = PyThread_allocate_lock(); - if (pending_lock == NULL) { - return _PyStatus_NO_MEMORY(); - } - /* We completely serialize creation of multiple interpreters, since it simplifies things here and blocking concurrent calls isn't a problem. Regardless, we must fully block subinterpreter creation until @@ -781,11 +699,10 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) interpreters->head = interp; status = init_interpreter(interp, runtime, - id, old_head, pending_lock); + id, old_head); if (_PyStatus_EXCEPTION(status)) { goto error; } - pending_lock = NULL; HEAD_UNLOCK(runtime); @@ -796,9 +713,6 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) error: HEAD_UNLOCK(runtime); - if (pending_lock != NULL) { - PyThread_free_lock(pending_lock); - } if (interp != NULL) { free_interpreter(interp); } @@ -1003,8 +917,6 @@ PyInterpreterState_Delete(PyInterpreterState *interp) zapthreads(interp); - _PyEval_FiniState(&interp->ceval); - // XXX These two calls should be done at the end of clear_interpreter(), // but currently some objects get decref'ed after that. #ifdef Py_REF_DEBUG @@ -1309,7 +1221,7 @@ _PyInterpreterState_LookUpID(int64_t requested_id) HEAD_UNLOCK(runtime); } if (interp == NULL && !PyErr_Occurred()) { - PyErr_Format(PyExc_RuntimeError, + PyErr_Format(PyExc_InterpreterNotFoundError, "unrecognized interpreter ID %lld", requested_id); } return interp; @@ -1353,20 +1265,19 @@ allocate_chunk(int size_in_bytes, _PyStackChunk* previous) return res; } -static PyThreadState * +static _PyThreadStateImpl * alloc_threadstate(void) { - return PyMem_RawCalloc(1, sizeof(PyThreadState)); + return PyMem_RawCalloc(1, sizeof(_PyThreadStateImpl)); } static void -free_threadstate(PyThreadState *tstate) +free_threadstate(_PyThreadStateImpl *tstate) { // The initial thread state of the interpreter is allocated // as part of the interpreter state so should not be freed. - if (tstate == &tstate->interp->_initial_thread) { + if (tstate == &tstate->base.interp->_initial_thread) { // Restore to _PyThreadState_INIT. - tstate = &tstate->interp->_initial_thread; memcpy(tstate, &initial._main_interpreter._initial_thread, sizeof(*tstate)); @@ -1385,9 +1296,10 @@ free_threadstate(PyThreadState *tstate) */ static void -init_threadstate(PyThreadState *tstate, +init_threadstate(_PyThreadStateImpl *_tstate, PyInterpreterState *interp, uint64_t id, int whence) { + PyThreadState *tstate = (PyThreadState *)_tstate; if (tstate->_status.initialized) { Py_FatalError("thread state already initialized"); } @@ -1444,13 +1356,13 @@ add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, static PyThreadState * new_threadstate(PyInterpreterState *interp, int whence) { - PyThreadState *tstate; + _PyThreadStateImpl *tstate; _PyRuntimeState *runtime = interp->runtime; // We don't need to allocate a thread state for the main interpreter // (the common case), but doing it later for the other case revealed a // reentrancy problem (deadlock). So for now we always allocate before // taking the interpreters lock. See GH-96071. - PyThreadState *new_tstate = alloc_threadstate(); + _PyThreadStateImpl *new_tstate = alloc_threadstate(); int used_newtstate; if (new_tstate == NULL) { return NULL; @@ -1482,14 +1394,14 @@ new_threadstate(PyInterpreterState *interp, int whence) } init_threadstate(tstate, interp, id, whence); - add_threadstate(interp, tstate, old_head); + add_threadstate(interp, (PyThreadState *)tstate, old_head); HEAD_UNLOCK(runtime); if (!used_newtstate) { // Must be called with lock unlocked to avoid re-entrancy deadlock. PyMem_RawFree(new_tstate); } - return tstate; + return (PyThreadState *)tstate; } PyThreadState * @@ -1547,6 +1459,7 @@ void PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); + assert(current_fast_get(&_PyRuntime)->interp == tstate->interp); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -1625,6 +1538,8 @@ PyThreadState_Clear(PyThreadState *tstate) tstate->on_delete(tstate->on_delete_data); } + _PyThreadState_ClearMimallocHeaps(tstate); + tstate->_status.cleared = 1; // XXX Call _PyThreadStateSwap(runtime, NULL) here if "current". @@ -1678,7 +1593,7 @@ zapthreads(PyInterpreterState *interp) while ((tstate = interp->threads.head) != NULL) { tstate_verify_not_active(tstate); tstate_delete_common(tstate); - free_threadstate(tstate); + free_threadstate((_PyThreadStateImpl *)tstate); } } @@ -1689,7 +1604,7 @@ PyThreadState_Delete(PyThreadState *tstate) _Py_EnsureTstateNotNULL(tstate); tstate_verify_not_active(tstate); tstate_delete_common(tstate); - free_threadstate(tstate); + free_threadstate((_PyThreadStateImpl *)tstate); } @@ -1701,7 +1616,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) tstate_delete_common(tstate); current_fast_clear(tstate->interp->runtime); _PyEval_ReleaseLock(tstate->interp, NULL); - free_threadstate(tstate); + free_threadstate((_PyThreadStateImpl *)tstate); } void @@ -1751,7 +1666,7 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate) for (p = list; p; p = next) { next = p->next; PyThreadState_Clear(p); - free_threadstate(p); + free_threadstate((_PyThreadStateImpl *)p); } } @@ -2043,6 +1958,20 @@ _PyThreadState_Bind(PyThreadState *tstate) } } +#if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +uintptr_t +_Py_GetThreadLocal_Addr(void) +{ +#ifdef HAVE_THREAD_LOCAL + // gh-112535: Use the address of the thread-local PyThreadState variable as + // a unique identifier for the current thread. Each thread has a unique + // _Py_tss_tstate variable with a unique address. + return (uintptr_t)&_Py_tss_tstate; +#else +# error "no supported thread-local variable storage classifier" +#endif +} +#endif /***********************************/ /* routines for advanced debuggers */ @@ -2243,7 +2172,7 @@ _PyGILState_Fini(PyInterpreterState *interp) // XXX Drop this. -PyStatus +void _PyGILState_SetTstate(PyThreadState *tstate) { /* must init with valid states */ @@ -2253,7 +2182,7 @@ _PyGILState_SetTstate(PyThreadState *tstate) if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, PyGILState is shared by all interpreters. The main * interpreter is responsible to initialize it. */ - return _PyStatus_OK(); + return; } #ifndef NDEBUG @@ -2263,8 +2192,6 @@ _PyGILState_SetTstate(PyThreadState *tstate) assert(gilstate_tss_get(runtime) == tstate); assert(tstate->gilstate_counter == 1); #endif - - return _PyStatus_OK(); } PyInterpreterState * @@ -2589,3 +2516,51 @@ _PyThreadState_MustExit(PyThreadState *tstate) } return 1; } + +/********************/ +/* mimalloc support */ +/********************/ + +static void +tstate_mimalloc_bind(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + struct _mimalloc_thread_state *mts = &((_PyThreadStateImpl*)tstate)->mimalloc; + + // Initialize the mimalloc thread state. This must be called from the + // same thread that will use the thread state. The "mem" heap doubles as + // the "backing" heap. + mi_tld_t *tld = &mts->tld; + _mi_tld_init(tld, &mts->heaps[_Py_MIMALLOC_HEAP_MEM]); + + // Initialize each heap + for (Py_ssize_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + _mi_heap_init_ex(&mts->heaps[i], tld, _mi_arena_id_none()); + } + + // By default, object allocations use _Py_MIMALLOC_HEAP_OBJECT. + // _PyObject_GC_New() and similar functions temporarily override this to + // use one of the GC heaps. + mts->current_object_heap = &mts->heaps[_Py_MIMALLOC_HEAP_OBJECT]; +#endif +} + +void +_PyThreadState_ClearMimallocHeaps(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + if (!tstate->_status.bound) { + // The mimalloc heaps are only initialized when the thread is bound. + return; + } + + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + for (Py_ssize_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + // Abandon all segments in use by this thread. This pushes them to + // a shared pool to later be reclaimed by other threads. It's important + // to do this before the thread state is destroyed so that objects + // remain visible to the GC. + _mi_heap_collect_abandon(&tstate_impl->mimalloc.heaps[i]); + } +#endif +} diff --git a/Python/pytime.c b/Python/pytime.c index e4813d4a9c2a2a..77cb95f8feb179 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -55,6 +55,43 @@ #endif +static _PyTime_t +_PyTime_GCD(_PyTime_t x, _PyTime_t y) +{ + // Euclidean algorithm + assert(x >= 1); + assert(y >= 1); + while (y != 0) { + _PyTime_t tmp = y; + y = x % y; + x = tmp; + } + assert(x >= 1); + return x; +} + + +int +_PyTimeFraction_Set(_PyTimeFraction *frac, _PyTime_t numer, _PyTime_t denom) +{ + if (numer < 1 || denom < 1) { + return -1; + } + + _PyTime_t gcd = _PyTime_GCD(numer, denom); + frac->numer = numer / gcd; + frac->denom = denom / gcd; + return 0; +} + + +double +_PyTimeFraction_Resolution(const _PyTimeFraction *frac) +{ + return (double)frac->numer / (double)frac->denom / 1e9; +} + + static void pytime_time_t_overflow(void) { @@ -152,11 +189,17 @@ _PyTime_Mul(_PyTime_t t, _PyTime_t k) } - - _PyTime_t -_PyTime_MulDiv(_PyTime_t ticks, _PyTime_t mul, _PyTime_t div) +_PyTimeFraction_Mul(_PyTime_t ticks, const _PyTimeFraction *frac) { + const _PyTime_t mul = frac->numer; + const _PyTime_t div = frac->denom; + + if (div == 1) { + // Fast-path taken by mach_absolute_time() with 1/1 time base. + return _PyTime_Mul(ticks, mul); + } + /* Compute (ticks * mul / div) in two parts to reduce the risk of integer overflow: compute the integer part, and then the remaining part. @@ -1016,51 +1059,34 @@ _PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) #ifdef __APPLE__ static int -py_mach_timebase_info(_PyTime_t *pnumer, _PyTime_t *pdenom, int raise) +py_mach_timebase_info(_PyTimeFraction *base, int raise) { - static mach_timebase_info_data_t timebase; - /* According to the Technical Q&A QA1398, mach_timebase_info() cannot - fail: https://developer.apple.com/library/mac/#qa/qa1398/ */ + mach_timebase_info_data_t timebase; + // According to the Technical Q&A QA1398, mach_timebase_info() cannot + // fail: https://developer.apple.com/library/mac/#qa/qa1398/ (void)mach_timebase_info(&timebase); - /* Sanity check: should never occur in practice */ - if (timebase.numer < 1 || timebase.denom < 1) { + // Check that timebase.numer and timebase.denom can be casted to + // _PyTime_t. In practice, timebase uses uint32_t, so casting cannot + // overflow. At the end, only make sure that the type is uint32_t + // (_PyTime_t is 64-bit long). + Py_BUILD_ASSERT(sizeof(timebase.numer) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(timebase.denom) <= sizeof(_PyTime_t)); + _PyTime_t numer = (_PyTime_t)timebase.numer; + _PyTime_t denom = (_PyTime_t)timebase.denom; + + // Known time bases: + // + // * (1, 1) on Intel: 1 ns + // * (1000000000, 33333335) on PowerPC: ~30 ns + // * (1000000000, 25000000) on PowerPC: 40 ns + if (_PyTimeFraction_Set(base, numer, denom) < 0) { if (raise) { PyErr_SetString(PyExc_RuntimeError, "invalid mach_timebase_info"); } return -1; } - - /* Check that timebase.numer and timebase.denom can be casted to - _PyTime_t. In practice, timebase uses uint32_t, so casting cannot - overflow. At the end, only make sure that the type is uint32_t - (_PyTime_t is 64-bit long). */ - static_assert(sizeof(timebase.numer) <= sizeof(_PyTime_t), - "timebase.numer is larger than _PyTime_t"); - static_assert(sizeof(timebase.denom) <= sizeof(_PyTime_t), - "timebase.denom is larger than _PyTime_t"); - - /* Make sure that _PyTime_MulDiv(ticks, timebase_numer, timebase_denom) - cannot overflow. - - Known time bases: - - * (1, 1) on Intel - * (1000000000, 33333335) or (1000000000, 25000000) on PowerPC - - None of these time bases can overflow with 64-bit _PyTime_t, but - check for overflow, just in case. */ - if ((_PyTime_t)timebase.numer > _PyTime_MAX / (_PyTime_t)timebase.denom) { - if (raise) { - PyErr_SetString(PyExc_OverflowError, - "mach_timebase_info is too large"); - } - return -1; - } - - *pnumer = (_PyTime_t)timebase.numer; - *pdenom = (_PyTime_t)timebase.denom; return 0; } #endif @@ -1109,17 +1135,16 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } #elif defined(__APPLE__) - static _PyTime_t timebase_numer = 0; - static _PyTime_t timebase_denom = 0; - if (timebase_denom == 0) { - if (py_mach_timebase_info(&timebase_numer, &timebase_denom, raise_exc) < 0) { + static _PyTimeFraction base = {0, 0}; + if (base.denom == 0) { + if (py_mach_timebase_info(&base, raise_exc) < 0) { return -1; } } if (info) { info->implementation = "mach_absolute_time()"; - info->resolution = (double)timebase_numer / (double)timebase_denom * 1e-9; + info->resolution = _PyTimeFraction_Resolution(&base); info->monotonic = 1; info->adjustable = 0; } @@ -1129,7 +1154,7 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) assert(uticks <= (uint64_t)_PyTime_MAX); _PyTime_t ticks = (_PyTime_t)uticks; - _PyTime_t ns = _PyTime_MulDiv(ticks, timebase_numer, timebase_denom); + _PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); *tp = pytime_from_nanoseconds(ns); #elif defined(__hpux) @@ -1213,7 +1238,7 @@ _PyTime_GetMonotonicClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #ifdef MS_WINDOWS static int -py_win_perf_counter_frequency(LONGLONG *pfrequency, int raise) +py_win_perf_counter_frequency(_PyTimeFraction *base, int raise) { LONGLONG frequency; @@ -1225,25 +1250,20 @@ py_win_perf_counter_frequency(LONGLONG *pfrequency, int raise) // Since Windows XP, frequency cannot be zero. assert(frequency >= 1); - /* Make also sure that (ticks * SEC_TO_NS) cannot overflow in - _PyTime_MulDiv(), with ticks < frequency. + Py_BUILD_ASSERT(sizeof(_PyTime_t) == sizeof(frequency)); + _PyTime_t denom = (_PyTime_t)frequency; - Known QueryPerformanceFrequency() values: - - * 10,000,000 (10 MHz): 100 ns resolution - * 3,579,545 Hz (3.6 MHz): 279 ns resolution - - None of these frequencies can overflow with 64-bit _PyTime_t, but - check for integer overflow just in case. */ - if (frequency > _PyTime_MAX / SEC_TO_NS) { + // Known QueryPerformanceFrequency() values: + // + // * 10,000,000 (10 MHz): 100 ns resolution + // * 3,579,545 Hz (3.6 MHz): 279 ns resolution + if (_PyTimeFraction_Set(base, SEC_TO_NS, denom) < 0) { if (raise) { - PyErr_SetString(PyExc_OverflowError, - "QueryPerformanceFrequency is too large"); + PyErr_SetString(PyExc_RuntimeError, + "invalid QueryPerformanceFrequency"); } return -1; } - - *pfrequency = frequency; return 0; } @@ -1253,16 +1273,16 @@ py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); - static LONGLONG frequency = 0; - if (frequency == 0) { - if (py_win_perf_counter_frequency(&frequency, raise_exc) < 0) { + static _PyTimeFraction base = {0, 0}; + if (base.denom == 0) { + if (py_win_perf_counter_frequency(&base, raise_exc) < 0) { return -1; } } if (info) { info->implementation = "QueryPerformanceCounter()"; - info->resolution = 1.0 / (double)frequency; + info->resolution = _PyTimeFraction_Resolution(&base); info->monotonic = 1; info->adjustable = 0; } @@ -1278,7 +1298,7 @@ py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) "LONGLONG is larger than _PyTime_t"); ticks = (_PyTime_t)ticksll; - _PyTime_t ns = _PyTime_MulDiv(ticks, SEC_TO_NS, (_PyTime_t)frequency); + _PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); *tp = pytime_from_nanoseconds(ns); return 0; } diff --git a/Python/specialize.c b/Python/specialize.c index ba704cbbb464d7..369b962a545f4e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -10,6 +10,7 @@ #include "pycore_moduleobject.h" #include "pycore_object.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Caches +#include "pycore_uop_metadata.h" // _PyOpcode_uop_name #include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _Py_ID() @@ -233,6 +234,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); + fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); print_histogram(out, "Trace length", stats->trace_length_hist); print_histogram(out, "Trace run length", stats->trace_run_length_hist); diff --git a/Python/symtable.c b/Python/symtable.c index da7fec0ee7cf0c..52d5932896b263 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -497,18 +497,14 @@ _PySymtable_Lookup(struct symtable *st, void *key) k = PyLong_FromVoidPtr(key); if (k == NULL) return NULL; - v = PyDict_GetItemWithError(st->st_blocks, k); - Py_DECREF(k); - - if (v) { - assert(PySTEntry_Check(v)); - } - else if (!PyErr_Occurred()) { + if (PyDict_GetItemRef(st->st_blocks, k, &v) == 0) { PyErr_SetString(PyExc_KeyError, "unknown symbol table entry"); } + Py_DECREF(k); - return (PySTEntryObject *)Py_XNewRef(v); + assert(v == NULL || PySTEntry_Check(v)); + return (PySTEntryObject *)v; } long diff --git a/Python/sysmodule.c b/Python/sysmodule.c index c17de44731b703..c2de4ecdc8ce0f 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -451,15 +451,9 @@ PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) e->hookCFunction = (Py_AuditHookFunction)hook; e->userData = userData; - if (runtime->audit_hooks.mutex == NULL) { - /* The runtime must not be initialized yet. */ - add_audit_hook_entry_unlocked(runtime, e); - } - else { - PyThread_acquire_lock(runtime->audit_hooks.mutex, WAIT_LOCK); - add_audit_hook_entry_unlocked(runtime, e); - PyThread_release_lock(runtime->audit_hooks.mutex); - } + PyMutex_Lock(&runtime->audit_hooks.mutex); + add_audit_hook_entry_unlocked(runtime, e); + PyMutex_Unlock(&runtime->audit_hooks.mutex); return 0; } @@ -989,6 +983,23 @@ sys_intern_impl(PyObject *module, PyObject *s) } +/*[clinic input] +sys._is_interned -> bool + + string: unicode + / + +Return True if the given string is "interned". +[clinic start generated code]*/ + +static int +sys__is_interned_impl(PyObject *module, PyObject *string) +/*[clinic end generated code: output=c3678267b4e9d7ed input=039843e17883b606]*/ +{ + return PyUnicode_CHECK_INTERNED(string); +} + + /* * Cached interned string objects used for calling the profile and * trace functions. @@ -1704,6 +1715,13 @@ static PyObject * sys__enablelegacywindowsfsencoding_impl(PyObject *module) /*[clinic end generated code: output=f5c3855b45e24fe9 input=2bfa931a20704492]*/ { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "sys._enablelegacywindowsfsencoding() is deprecated and will be " + "removed in Python 3.16. Use PYTHONLEGACYWINDOWSFSENCODING " + "instead.", 1)) + { + return NULL; + } if (_PyUnicode_EnableLegacyWindowsFSEncoding() < 0) { return NULL; } @@ -2462,6 +2480,7 @@ static PyMethodDef sys_methods[] = { SYS_GETWINDOWSVERSION_METHODDEF SYS__ENABLELEGACYWINDOWSFSENCODING_METHODDEF SYS_INTERN_METHODDEF + SYS__IS_INTERNED_METHODDEF SYS_IS_FINALIZING_METHODDEF SYS_MDEBUG_METHODDEF SYS_SETSWITCHINTERVAL_METHODDEF diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 218c64e13374e6..05633e3f77af49 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -21,7 +21,7 @@ verbose = False -# This must be kept in sync with Tools/cases_generator/generate_cases.py +# This must be kept in sync with Tools/cases_generator/analyzer.py RESUME = 149 def isprintable(b: bytes) -> bool: diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index c5a397129201b6..a541b4b33c519b 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -468,6 +468,17 @@ def replace_block(lines, start_marker, end_marker, replacements, file): return lines[:start_pos + 1] + replacements + lines[end_pos:] +class UniqueList(list): + def __init__(self): + self._seen = set() + + def append(self, item): + if item in self._seen: + return + super().append(item) + self._seen.add(item) + + def regen_frozen(modules): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) @@ -477,7 +488,7 @@ def regen_frozen(modules): header = relpath_for_posix_display(src.frozenfile, parentdir) headerlines.append(f'#include "{header}"') - externlines = [] + externlines = UniqueList() bootstraplines = [] stdliblines = [] testlines = [] @@ -647,7 +658,7 @@ def regen_pcbuild(modules): filterlines = [] corelines = [] deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}' - deepfreezerules = [f' '] + deepfreezerules = [f' '] deepfreezemappings = [] for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) @@ -656,15 +667,15 @@ def regen_pcbuild(modules): projlines.append(f' ') projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') - projlines.append(f' $(PySourcePath){header}') + projlines.append(f' $(GeneratedFrozenModulesDir){header}') projlines.append(f' ') filterlines.append(f' ') filterlines.append(' Python Files') filterlines.append(' ') - deepfreezemappings.append(f' \n') + deepfreezemappings.append(f' \n') - corelines.append(f' ') + corelines.append(f' ') print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py new file mode 100644 index 00000000000000..93d0d8a3762df3 --- /dev/null +++ b/Tools/build/generate_sbom.py @@ -0,0 +1,275 @@ +"""Tool for generating Software Bill of Materials (SBOM) for Python's dependencies""" +import os +import re +import hashlib +import json +import glob +import pathlib +import subprocess +import sys +import typing +from urllib.request import urlopen + +CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent + +# Before adding a new entry to this list, double check that +# the license expression is a valid SPDX license expression: +# See: https://spdx.org/licenses +ALLOWED_LICENSE_EXPRESSIONS = { + "MIT", + "CC0-1.0", + "Apache-2.0", + "BSD-2-Clause", +} + +# Properties which are required for our purposes. +REQUIRED_PROPERTIES_PACKAGE = frozenset([ + "SPDXID", + "name", + "versionInfo", + "downloadLocation", + "checksums", + "licenseConcluded", + "externalRefs", + "originator", + "primaryPackagePurpose", +]) + + +class PackageFiles(typing.NamedTuple): + """Structure for describing the files of a package""" + include: list[str] + exclude: list[str] | None = None + + +# SBOMS don't have a method to specify the sources of files +# so we need to do that external to the SBOM itself. Add new +# values to 'exclude' if we create new files within tracked +# directories that aren't sourced from third-party packages. +PACKAGE_TO_FILES = { + # NOTE: pip's entry in this structure is automatically generated in + # the 'discover_pip_sbom_package()' function below. + "mpdecimal": PackageFiles( + include=["Modules/_decimal/libmpdec/**"] + ), + "expat": PackageFiles( + include=["Modules/expat/**"] + ), + "macholib": PackageFiles( + include=["Lib/ctypes/macholib/**"], + exclude=[ + "Lib/ctypes/macholib/README.ctypes", + "Lib/ctypes/macholib/fetch_macholib", + "Lib/ctypes/macholib/fetch_macholib.bat", + ], + ), + "libb2": PackageFiles( + include=["Modules/_blake2/impl/**"] + ), + "hacl-star": PackageFiles( + include=["Modules/_hacl/**"], + exclude=[ + "Modules/_hacl/refresh.sh", + "Modules/_hacl/README.md", + "Modules/_hacl/python_hacl_namespace.h", + ] + ), +} + + +def spdx_id(value: str) -> str: + """Encode a value into characters that are valid in an SPDX ID""" + return re.sub(r"[^a-zA-Z0-9.\-]+", "-", value) + + +def filter_gitignored_paths(paths: list[str]) -> list[str]: + """ + Filter out paths excluded by the gitignore file. + The output of 'git check-ignore --non-matching --verbose' looks + like this for non-matching (included) files: + + '::' + + And looks like this for matching (excluded) files: + + '.gitignore:9:*.a Tools/lib.a' + """ + # Filter out files in gitignore. + # Non-matching files show up as '::' + git_check_ignore_proc = subprocess.run( + ["git", "check-ignore", "--verbose", "--non-matching", *paths], + check=False, + stdout=subprocess.PIPE, + ) + # 1 means matches, 0 means no matches. + assert git_check_ignore_proc.returncode in (0, 1) + + # Return the list of paths sorted + git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines() + return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) + + +def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: + """pip is a part of a packaging ecosystem (Python, surprise!) so it's actually + automatable to discover the metadata we need like the version and checksums + so let's do that on behalf of our friends at the PyPA. + """ + global PACKAGE_TO_FILES + + ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled" + pip_wheels = [] + + # Find the hopefully one pip wheel in the bundled directory. + for wheel_filename in os.listdir(ensurepip_bundled_dir): + if wheel_filename.startswith("pip-"): + pip_wheels.append(wheel_filename) + if len(pip_wheels) != 1: + print("Zero or multiple pip wheels detected in 'Lib/ensurepip/_bundled'") + sys.exit(1) + pip_wheel_filename = pip_wheels[0] + + # Add the wheel filename to the list of files so the SBOM file + # and relationship generator can work its magic on the wheel too. + PACKAGE_TO_FILES["pip"] = PackageFiles( + include=[f"Lib/ensurepip/_bundled/{pip_wheel_filename}"] + ) + + # Wheel filename format puts the version right after the project name. + pip_version = pip_wheel_filename.split("-")[1] + pip_checksum_sha256 = hashlib.sha256( + (ensurepip_bundled_dir / pip_wheel_filename).read_bytes() + ).hexdigest() + + # Get pip's download location from PyPI. Check that the checksum is correct too. + try: + raw_text = urlopen(f"https://pypi.org/pypi/pip/{pip_version}/json").read() + pip_release_metadata = json.loads(raw_text) + url: dict[str, typing.Any] + + # Look for a matching artifact filename and then check + # its remote checksum to the local one. + for url in pip_release_metadata["urls"]: + if url["filename"] == pip_wheel_filename: + break + else: + raise ValueError(f"No matching filename on PyPI for '{pip_wheel_filename}'") + if url["digests"]["sha256"] != pip_checksum_sha256: + raise ValueError(f"Local pip checksum doesn't match artifact on PyPI") + + # Successfully found the download URL for the matching artifact. + pip_download_url = url["url"] + + except (OSError, ValueError) as e: + print(f"Couldn't fetch pip's metadata from PyPI: {e}") + sys.exit(1) + + # Remove pip from the existing SBOM packages if it's there + # and then overwrite its entry with our own generated one. + sbom_data["packages"] = [ + sbom_package + for sbom_package in sbom_data["packages"] + if sbom_package["name"] != "pip" + ] + sbom_data["packages"].append( + { + "SPDXID": spdx_id("SPDXRef-PACKAGE-pip"), + "name": "pip", + "versionInfo": pip_version, + "originator": "Organization: Python Packaging Authority", + "licenseConcluded": "MIT", + "downloadLocation": pip_download_url, + "checksums": [ + {"algorithm": "SHA256", "checksumValue": pip_checksum_sha256} + ], + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": f"cpe:2.3:a:pypa:pip:{pip_version}:*:*:*:*:*:*:*", + "referenceType": "cpe23Type", + }, + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": f"pkg:pypi/pip@{pip_version}", + "referenceType": "purl", + }, + ], + "primaryPackagePurpose": "SOURCE", + } + ) + + +def main() -> None: + sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" + sbom_data = json.loads(sbom_path.read_bytes()) + + # Insert pip's SBOM metadata from the wheel. + discover_pip_sbom_package(sbom_data) + + # Ensure all packages in this tool are represented also in the SBOM file. + assert {package["name"] for package in sbom_data["packages"]} == set(PACKAGE_TO_FILES) + + # Make a bunch of assertions about the SBOM data to ensure it's consistent. + for package in sbom_data["packages"]: + + # Properties and ID must be properly formed. + assert set(package.keys()) == REQUIRED_PROPERTIES_PACKAGE + assert package["SPDXID"] == spdx_id(f"SPDXRef-PACKAGE-{package['name']}") + + # Version must be in the download and external references. + version = package["versionInfo"] + assert version in package["downloadLocation"] + assert all(version in ref["referenceLocator"] for ref in package["externalRefs"]) + + # License must be on the approved list for SPDX. + assert package["licenseConcluded"] in ALLOWED_LICENSE_EXPRESSIONS, package["licenseConcluded"] + + # Regenerate file information from current data. + sbom_files = [] + sbom_relationships = [] + + # We call 'sorted()' here a lot to avoid filesystem scan order issues. + for name, files in sorted(PACKAGE_TO_FILES.items()): + package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}") + exclude = files.exclude or () + for include in sorted(files.include): + + # Find all the paths and then filter them through .gitignore. + paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True) + paths = filter_gitignored_paths(paths) + assert paths, include # Make sure that every value returns something! + + for path in paths: + # Skip directories and excluded files + if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude: + continue + + # SPDX requires SHA1 to be used for files, but we provide SHA256 too. + data = (CPYTHON_ROOT_DIR / path).read_bytes() + checksum_sha1 = hashlib.sha1(data).hexdigest() + checksum_sha256 = hashlib.sha256(data).hexdigest() + + file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}") + sbom_files.append({ + "SPDXID": file_spdx_id, + "fileName": path, + "checksums": [ + {"algorithm": "SHA1", "checksumValue": checksum_sha1}, + {"algorithm": "SHA256", "checksumValue": checksum_sha256}, + ], + }) + + # Tie each file back to its respective package. + sbom_relationships.append({ + "spdxElementId": package_spdx_id, + "relatedSpdxElement": file_spdx_id, + "relationshipType": "CONTAINS", + }) + + # Update the SBOM on disk + sbom_data["files"] = sbom_files + sbom_data["relationships"] = sbom_relationships + sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 766a85d3d6f39e..5dce4e042d1eb4 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -36,6 +36,7 @@ '_testsinglephase', '_xxsubinterpreters', '_xxinterpchannels', + '_xxinterpqueues', '_xxtestfuzz', 'idlelib.idle_test', 'test', diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini new file mode 100644 index 00000000000000..cf1dac7fde5ac5 --- /dev/null +++ b/Tools/build/mypy.ini @@ -0,0 +1,13 @@ +[mypy] +files = Tools/build/generate_sbom.py +pretty = True + +# Make sure Python can still be built +# using Python 3.10 for `PYTHON_FOR_REGEN`... +python_version = 3.10 + +# ...And be strict: +strict = True +strict_concatenate = True +enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined +warn_unreachable = True diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 7cba788ff33578..85c437d521a15a 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -521,7 +521,7 @@ def gcc_get_limited_api_macros(headers): api_hexversion = sys.version_info.major << 24 | sys.version_info.minor << 16 - preprocesor_output_with_macros = subprocess.check_output( + preprocessor_output_with_macros = subprocess.check_output( sysconfig.get_config_var("CC").split() + [ # Prevent the expansion of the exported macros so we can @@ -540,7 +540,7 @@ def gcc_get_limited_api_macros(headers): return { target for target in re.findall( - r"#define (\w+)", preprocesor_output_with_macros + r"#define (\w+)", preprocessor_output_with_macros ) } @@ -561,7 +561,7 @@ def gcc_get_limited_api_definitions(headers): Requires Python built with a GCC-compatible compiler. (clang might work) """ api_hexversion = sys.version_info.major << 24 | sys.version_info.minor << 16 - preprocesor_output = subprocess.check_output( + preprocessor_output = subprocess.check_output( sysconfig.get_config_var("CC").split() + [ # Prevent the expansion of the exported macros so we can capture @@ -581,13 +581,13 @@ def gcc_get_limited_api_definitions(headers): stderr=subprocess.DEVNULL, ) stable_functions = set( - re.findall(r"__PyAPI_FUNC\(.*?\)\s*(.*?)\s*\(", preprocesor_output) + re.findall(r"__PyAPI_FUNC\(.*?\)\s*(.*?)\s*\(", preprocessor_output) ) stable_exported_data = set( - re.findall(r"__EXPORT_DATA\((.*?)\)", preprocesor_output) + re.findall(r"__EXPORT_DATA\((.*?)\)", preprocessor_output) ) stable_data = set( - re.findall(r"__PyAPI_DATA\(.*?\)[\s\*\(]*([^);]*)\)?.*;", preprocesor_output) + re.findall(r"__PyAPI_DATA\(.*?\)[\s\*\(]*([^);]*)\)?.*;", preprocessor_output) ) return stable_data | stable_exported_data | stable_functions diff --git a/Tools/c-analyzer/c_analyzer/__init__.py b/Tools/c-analyzer/c_analyzer/__init__.py index 171fa25102bffc..b83ffc087a08d8 100644 --- a/Tools/c-analyzer/c_analyzer/__init__.py +++ b/Tools/c-analyzer/c_analyzer/__init__.py @@ -18,7 +18,7 @@ def analyze(filenmes, **kwargs): - results = iter_analyis_results(filenames, **kwargs) + results = iter_analysis_results(filenames, **kwargs) return Analysis.from_results(results) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 04388fb54caa6c..239ed4e0266a75 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -84,7 +84,6 @@ def clean_lines(text): Python/frozen_modules/*.h Python/generated_cases.c.h Python/executor_cases.c.h -Python/abstract_interp_cases.c.h # not actually source Python/bytecodes.c diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index aa8ce49ae86376..e3a1b5d532bda2 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -290,6 +290,10 @@ Objects/exceptions.c - PyExc_UnicodeWarning - Objects/exceptions.c - PyExc_BytesWarning - Objects/exceptions.c - PyExc_ResourceWarning - Objects/exceptions.c - PyExc_EncodingWarning - +Python/crossinterp.c - _PyExc_InterpreterError - +Python/crossinterp.c - _PyExc_InterpreterNotFoundError - +Python/crossinterp.c - PyExc_InterpreterError - +Python/crossinterp.c - PyExc_InterpreterNotFoundError - ##----------------------- ## singletons diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index d59e0ddcdfde4e..2f9e80d6ab6737 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -165,6 +165,7 @@ Python/pylifecycle.c fatal_error reentrant - # explicitly protected, internal-only Modules/_xxinterpchannelsmodule.c - _globals - +Modules/_xxinterpqueuesmodule.c - _globals - # set once during module init Modules/_decimal/_decimal.c - minalloc_is_set - @@ -599,6 +600,9 @@ Modules/_xxtestfuzz/fuzzer.c - re_error_exception - Modules/_xxtestfuzz/fuzzer.c - struct_error - Modules/_xxtestfuzz/fuzzer.c - struct_unpack_method - Modules/_xxtestfuzz/fuzzer.c - xmlparser_type - +Modules/_xxtestfuzz/fuzzer.c - pycompile_scratch - +Modules/_xxtestfuzz/fuzzer.c - start_vals - +Modules/_xxtestfuzz/fuzzer.c - optimize_vals - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput CSV_READER_INITIALIZED - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput JSON_LOADS_INITIALIZED - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_COMPILE_INITIALIZED - diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py deleted file mode 100644 index 603b15596f16de..00000000000000 --- a/Tools/cases_generator/analysis.py +++ /dev/null @@ -1,482 +0,0 @@ -import re -import sys -import typing - -from _typing_backports import assert_never -from flags import InstructionFlags, variable_used -from formatting import prettify_filename, UNUSED -from instructions import ( - ActiveCacheEffect, - Component, - Instruction, - InstructionOrCacheEffect, - MacroInstruction, - MacroParts, - PseudoInstruction, -) -import parsing -from parsing import StackEffect - -BEGIN_MARKER = "// BEGIN BYTECODES //" -END_MARKER = "// END BYTECODES //" - -RESERVED_WORDS = { - "co_consts": "Use FRAME_CO_CONSTS.", - "co_names": "Use FRAME_CO_NAMES.", -} - -RE_GO_TO_INSTR = r"^\s*GO_TO_INSTRUCTION\((\w+)\);\s*(?://.*)?$" - - -class Analyzer: - """Parse input, analyze it, and write to output.""" - - input_filenames: list[str] - errors: int = 0 - warnings: int = 0 - - def __init__(self, input_filenames: list[str]): - self.input_filenames = input_filenames - - def message(self, msg: str, node: parsing.Node) -> None: - lineno = 0 - filename = "" - if context := node.context: - filename = context.owner.filename - # Use line number of first non-comment in the node - for token in context.owner.tokens[context.begin : context.end]: - lineno = token.line - if token.kind != "COMMENT": - break - print(f"{filename}:{lineno}: {msg}", file=sys.stderr) - - def error(self, msg: str, node: parsing.Node) -> None: - self.message("error: " + msg, node) - self.errors += 1 - - def warning(self, msg: str, node: parsing.Node) -> None: - self.message("warning: " + msg, node) - self.warnings += 1 - - def note(self, msg: str, node: parsing.Node) -> None: - self.message("note: " + msg, node) - - everything: list[ - parsing.InstDef - | parsing.Macro - | parsing.Pseudo - ] - instrs: dict[str, Instruction] # Includes ops - macros: dict[str, parsing.Macro] - macro_instrs: dict[str, MacroInstruction] - families: dict[str, parsing.Family] - pseudos: dict[str, parsing.Pseudo] - pseudo_instrs: dict[str, PseudoInstruction] - - def parse(self) -> None: - """Parse the source text. - - We only want the parser to see the stuff between the - begin and end markers. - """ - - self.everything = [] - self.instrs = {} - self.macros = {} - self.families = {} - self.pseudos = {} - - instrs_idx: dict[str, int] = dict() - - for filename in self.input_filenames: - self.parse_file(filename, instrs_idx) - - files = " + ".join(self.input_filenames) - n_instrs = len(set(self.instrs) & set(self.macros)) - n_ops = len(self.instrs) - n_instrs - print( - f"Read {n_instrs} instructions, {n_ops} ops, " - f"{len(self.macros)} macros, {len(self.pseudos)} pseudos, " - f"and {len(self.families)} families from {files}", - file=sys.stderr, - ) - - def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: - with open(filename) as file: - src = file.read() - - psr = parsing.Parser(src, filename=prettify_filename(filename)) - - # Skip until begin marker - while tkn := psr.next(raw=True): - if tkn.text == BEGIN_MARKER: - break - else: - raise psr.make_syntax_error( - f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" - ) - start = psr.getpos() - - # Find end marker, then delete everything after it - while tkn := psr.next(raw=True): - if tkn.text == END_MARKER: - break - del psr.tokens[psr.getpos() - 1 :] - - # Parse from start - psr.setpos(start) - thing: parsing.Node | None - thing_first_token = psr.peek() - while thing := psr.definition(): - thing = typing.cast( - parsing.InstDef | parsing.Macro | parsing.Pseudo | parsing.Family, thing - ) - if ws := [w for w in RESERVED_WORDS if variable_used(thing, w)]: - self.error( - f"'{ws[0]}' is a reserved word. {RESERVED_WORDS[ws[0]]}", thing - ) - - match thing: - case parsing.InstDef(name=name): - macro: parsing.Macro | None = None - if thing.kind == "inst" and "override" not in thing.annotations: - macro = parsing.Macro(name, [parsing.OpName(name)]) - if name in self.instrs: - if "override" not in thing.annotations: - raise psr.make_syntax_error( - f"Duplicate definition of '{name}' @ {thing.context} " - f"previous definition @ {self.instrs[name].inst.context}", - thing_first_token, - ) - self.everything[instrs_idx[name]] = thing - if name not in self.instrs and "override" in thing.annotations: - raise psr.make_syntax_error( - f"Definition of '{name}' @ {thing.context} is supposed to be " - "an override but no previous definition exists.", - thing_first_token, - ) - self.instrs[name] = Instruction(thing) - instrs_idx[name] = len(self.everything) - self.everything.append(thing) - if macro is not None: - self.macros[macro.name] = macro - self.everything.append(macro) - case parsing.Macro(name): - self.macros[name] = thing - self.everything.append(thing) - case parsing.Family(name): - self.families[name] = thing - case parsing.Pseudo(name): - self.pseudos[name] = thing - self.everything.append(thing) - case _: - assert_never(thing) - if not psr.eof(): - raise psr.make_syntax_error(f"Extra stuff at the end of {filename}") - - def analyze(self) -> None: - """Analyze the inputs. - - Raises SystemExit if there is an error. - """ - self.analyze_macros_and_pseudos() - self.map_families() - self.mark_predictions() - self.check_families() - - def mark_predictions(self) -> None: - """Mark the instructions that need PREDICTED() labels.""" - # Start with family heads - for family in self.families.values(): - if family.name in self.instrs: - self.instrs[family.name].predicted = True - if family.name in self.macro_instrs: - self.macro_instrs[family.name].predicted = True - # Also look for GO_TO_INSTRUCTION() calls - for instr in self.instrs.values(): - targets: set[str] = set() - for line in instr.block_text: - if m := re.match(RE_GO_TO_INSTR, line): - targets.add(m.group(1)) - for target in targets: - if target_instr := self.instrs.get(target): - target_instr.predicted = True - if target_macro := self.macro_instrs.get(target): - target_macro.predicted = True - if not target_instr and not target_macro: - self.error( - f"Unknown instruction {target!r} predicted in {instr.name!r}", - instr.inst, # TODO: Use better location - ) - - def map_families(self) -> None: - """Link instruction names back to their family, if they have one.""" - for family in self.families.values(): - for member in [family.name] + family.members: - if member_instr := self.instrs.get(member): - if ( - member_instr.family is not family - and member_instr.family is not None - ): - self.error( - f"Instruction {member} is a member of multiple families " - f"({member_instr.family.name}, {family.name}).", - family, - ) - else: - member_instr.family = family - if member_mac := self.macro_instrs.get(member): - assert member_mac.family is None, (member, member_mac.family.name) - member_mac.family = family - if not member_instr and not member_mac: - self.error( - f"Unknown instruction {member!r} referenced in family {family.name!r}", - family, - ) - # A sanctioned exception: - # This opcode is a member of the family but it doesn't pass the checks. - if mac := self.macro_instrs.get("BINARY_OP_INPLACE_ADD_UNICODE"): - mac.family = self.families.get("BINARY_OP") - - def check_families(self) -> None: - """Check each family: - - - Must have at least 2 members (including head) - - Head and all members must be known instructions - - Head and all members must have the same cache, input and output effects - """ - for family in self.families.values(): - if family.name not in self.macro_instrs and family.name not in self.instrs: - self.error( - f"Family {family.name!r} has unknown instruction {family.name!r}", - family, - ) - members = [ - member - for member in family.members - if member in self.instrs or member in self.macro_instrs - ] - if members != family.members: - unknown = set(family.members) - set(members) - self.error( - f"Family {family.name!r} has unknown members: {unknown}", family - ) - expected_effects = self.effect_counts(family.name) - for member in members: - member_effects = self.effect_counts(member) - if member_effects != expected_effects: - self.error( - f"Family {family.name!r} has inconsistent " - f"(cache, input, output) effects:\n" - f" {family.name} = {expected_effects}; " - f"{member} = {member_effects}", - family, - ) - - def effect_counts(self, name: str) -> tuple[int, int, int]: - if mac := self.macro_instrs.get(name): - cache = mac.cache_offset - input, output = 0, 0 - for part in mac.parts: - if isinstance(part, Component): - # A component may pop what the previous component pushed, - # so we offset the input/output counts by that. - delta_i = len(part.instr.input_effects) - delta_o = len(part.instr.output_effects) - offset = min(delta_i, output) - input += delta_i - offset - output += delta_o - offset - else: - assert False, f"Unknown instruction {name!r}" - return cache, input, output - - def analyze_macros_and_pseudos(self) -> None: - """Analyze each macro and pseudo instruction.""" - self.macro_instrs = {} - self.pseudo_instrs = {} - for name, macro in self.macros.items(): - self.macro_instrs[name] = mac = self.analyze_macro(macro) - self.check_macro_consistency(mac) - for name, pseudo in self.pseudos.items(): - self.pseudo_instrs[name] = self.analyze_pseudo(pseudo) - - # TODO: Merge with similar code in stacking.py, write_components() - def check_macro_consistency(self, mac: MacroInstruction) -> None: - def get_var_names(instr: Instruction) -> dict[str, StackEffect]: - vars: dict[str, StackEffect] = {} - for eff in instr.input_effects + instr.output_effects: - if eff.name == UNUSED: - continue - if eff.name in vars: - if vars[eff.name] != eff: - self.error( - f"Instruction {instr.name!r} has " - f"inconsistent type/cond/size for variable " - f"{eff.name!r}: {vars[eff.name]} vs {eff}", - instr.inst, - ) - else: - vars[eff.name] = eff - return vars - - all_vars: dict[str, StackEffect] = {} - # print("Checking", mac.name) - prevop: Instruction | None = None - for part in mac.parts: - if not isinstance(part, Component): - continue - vars = get_var_names(part.instr) - # print(" //", part.instr.name, "//", vars) - for name, eff in vars.items(): - if name in all_vars: - if all_vars[name] != eff: - self.error( - f"Macro {mac.name!r} has " - f"inconsistent type/cond/size for variable " - f"{name!r}: " - f"{all_vars[name]} vs {eff} in {part.instr.name!r}", - mac.macro, - ) - else: - all_vars[name] = eff - if prevop is not None: - pushes = list(prevop.output_effects) - pops = list(reversed(part.instr.input_effects)) - copies: list[tuple[StackEffect, StackEffect]] = [] - while pushes and pops and pushes[-1] == pops[0]: - src, dst = pushes.pop(), pops.pop(0) - if src.name == dst.name or dst.name == UNUSED: - continue - copies.append((src, dst)) - reads = set(copy[0].name for copy in copies) - writes = set(copy[1].name for copy in copies) - if reads & writes: - self.error( - f"Macro {mac.name!r} has conflicting copies " - f"(source of one copy is destination of another): " - f"{reads & writes}", - mac.macro, - ) - prevop = part.instr - - def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: - components = self.check_macro_components(macro) - parts: MacroParts = [] - flags = InstructionFlags.newEmpty() - offset = 0 - for component in components: - match component: - case parsing.CacheEffect() as ceffect: - parts.append(ceffect) - offset += ceffect.size - case Instruction() as instr: - part, offset = self.analyze_instruction(instr, offset) - parts.append(part) - if instr.name != "_SAVE_RETURN_OFFSET": - # _SAVE_RETURN_OFFSET's oparg does not transfer - flags.add(instr.instr_flags) - case _: - assert_never(component) - format = "IB" if flags.HAS_ARG_FLAG else "IX" - if offset: - format += "C" + "0" * (offset - 1) - return MacroInstruction(macro.name, format, flags, macro, parts, offset) - - def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: - targets: list[Instruction | MacroInstruction] = [] - for target_name in pseudo.targets: - if target_name in self.instrs: - targets.append(self.instrs[target_name]) - else: - targets.append(self.macro_instrs[target_name]) - assert targets - ignored_flags = {"HAS_EVAL_BREAK_FLAG", "HAS_DEOPT_FLAG", "HAS_ERROR_FLAG", "HAS_ESCAPES_FLAG"} - assert len({t.instr_flags.bitmap(ignore=ignored_flags) for t in targets}) == 1 - return PseudoInstruction(pseudo.name, targets, targets[0].instr_flags) - - def analyze_instruction( - self, instr: Instruction, offset: int - ) -> tuple[Component, int]: - active_effects: list[ActiveCacheEffect] = [] - for ceffect in instr.cache_effects: - if ceffect.name != UNUSED: - active_effects.append(ActiveCacheEffect(ceffect, offset)) - offset += ceffect.size - return ( - Component(instr, active_effects), - offset, - ) - - def check_macro_components( - self, macro: parsing.Macro - ) -> list[InstructionOrCacheEffect]: - components: list[InstructionOrCacheEffect] = [] - for uop in macro.uops: - match uop: - case parsing.OpName(name): - if name not in self.instrs: - self.error(f"Unknown instruction {name!r}", macro) - else: - components.append(self.instrs[name]) - case parsing.CacheEffect(): - components.append(uop) - case _: - assert_never(uop) - return components - - def report_non_viable_uops(self, jsonfile: str) -> None: - print("The following ops are not viable uops:") - skips = { - "CACHE", - "RESERVED", - "INTERPRETER_EXIT", - "JUMP_BACKWARD", - "LOAD_FAST_LOAD_FAST", - "LOAD_CONST_LOAD_FAST", - "STORE_FAST_STORE_FAST", - "POP_JUMP_IF_TRUE", - "POP_JUMP_IF_FALSE", - "_ITER_JUMP_LIST", - "_ITER_JUMP_TUPLE", - "_ITER_JUMP_RANGE", - } - try: - # Secret feature: if bmraw.json exists, print and sort by execution count - counts = load_execution_counts(jsonfile) - except FileNotFoundError as err: - counts = {} - non_viable = [ - instr - for instr in self.instrs.values() - if instr.name not in skips - and not instr.name.startswith("INSTRUMENTED_") - and not instr.is_viable_uop() - ] - non_viable.sort(key=lambda instr: (-counts.get(instr.name, 0), instr.name)) - for instr in non_viable: - if instr.name in counts: - scount = f"{counts[instr.name]:,}" - else: - scount = "" - print(f" {scount:>15} {instr.name:<35}", end="") - if instr.name in self.families: - print(" (unspecialized)", end="") - elif instr.family is not None: - print(f" (specialization of {instr.family.name})", end="") - print() - - -def load_execution_counts(jsonfile: str) -> dict[str, int]: - import json - - with open(jsonfile) as f: - jsondata = json.load(f) - - # Look for keys like "opcode[LOAD_FAST].execution_count" - prefix = "opcode[" - suffix = "].execution_count" - res: dict[str, int] = {} - for key, value in jsondata.items(): - if key.startswith(prefix) and key.endswith(suffix): - res[key[len(prefix) : -len(suffix)]] = value - return res diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py new file mode 100644 index 00000000000000..82ef8888bfcee5 --- /dev/null +++ b/Tools/cases_generator/analyzer.py @@ -0,0 +1,723 @@ +from dataclasses import dataclass +import lexer +import parser +from typing import Optional + + +@dataclass +class Properties: + escapes: bool + infallible: bool + deopts: bool + oparg: bool + jumps: bool + eval_breaker: bool + ends_with_eval_breaker: bool + needs_this: bool + always_exits: bool + stores_sp: bool + tier_one_only: bool + uses_co_consts: bool + uses_co_names: bool + uses_locals: bool + has_free: bool + + def dump(self, indent: str) -> None: + print(indent, end="") + text = ", ".join([f"{key}: {value}" for (key, value) in self.__dict__.items()]) + print(indent, text, sep="") + + @staticmethod + def from_list(properties: list["Properties"]) -> "Properties": + return Properties( + escapes=any(p.escapes for p in properties), + infallible=all(p.infallible for p in properties), + deopts=any(p.deopts for p in properties), + oparg=any(p.oparg for p in properties), + jumps=any(p.jumps for p in properties), + eval_breaker=any(p.eval_breaker for p in properties), + ends_with_eval_breaker=any(p.ends_with_eval_breaker for p in properties), + needs_this=any(p.needs_this for p in properties), + always_exits=any(p.always_exits for p in properties), + stores_sp=any(p.stores_sp for p in properties), + tier_one_only=any(p.tier_one_only for p in properties), + uses_co_consts=any(p.uses_co_consts for p in properties), + uses_co_names=any(p.uses_co_names for p in properties), + uses_locals=any(p.uses_locals for p in properties), + has_free=any(p.has_free for p in properties), + ) + + +SKIP_PROPERTIES = Properties( + escapes=False, + infallible=True, + deopts=False, + oparg=False, + jumps=False, + eval_breaker=False, + ends_with_eval_breaker=False, + needs_this=False, + always_exits=False, + stores_sp=False, + tier_one_only=False, + uses_co_consts=False, + uses_co_names=False, + uses_locals=False, + has_free=False, +) + + +@dataclass +class Skip: + "Unused cache entry" + size: int + + @property + def name(self) -> str: + return f"unused/{self.size}" + + @property + def properties(self) -> Properties: + return SKIP_PROPERTIES + + +@dataclass +class StackItem: + name: str + type: str | None + condition: str | None + size: str + peek: bool = False + + def __str__(self) -> str: + cond = f" if ({self.condition})" if self.condition else "" + size = f"[{self.size}]" if self.size != "1" else "" + type = "" if self.type is None else f"{self.type} " + return f"{type}{self.name}{size}{cond} {self.peek}" + + def is_array(self) -> bool: + return self.type == "PyObject **" + + +@dataclass +class StackEffect: + inputs: list[StackItem] + outputs: list[StackItem] + + def __str__(self) -> str: + return f"({', '.join([str(i) for i in self.inputs])} -- {', '.join([str(i) for i in self.outputs])})" + + +@dataclass +class CacheEntry: + name: str + size: int + + def __str__(self) -> str: + return f"{self.name}/{self.size}" + + +@dataclass +class Uop: + name: str + context: parser.Context | None + annotations: list[str] + stack: StackEffect + caches: list[CacheEntry] + body: list[lexer.Token] + properties: Properties + _size: int = -1 + implicitly_created: bool = False + + def dump(self, indent: str) -> None: + print( + indent, self.name, ", ".join(self.annotations) if self.annotations else "" + ) + print(indent, self.stack, ", ".join([str(c) for c in self.caches])) + self.properties.dump(" " + indent) + + @property + def size(self) -> int: + if self._size < 0: + self._size = sum(c.size for c in self.caches) + return self._size + + def is_viable(self) -> bool: + if self.name == "_SAVE_RETURN_OFFSET": + return True # Adjusts next_instr, but only in tier 1 code + if self.properties.needs_this: + return False + if "INSTRUMENTED" in self.name: + return False + if "replaced" in self.annotations: + return False + if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"): + return False + if len([c for c in self.caches if c.name != "unused"]) > 1: + return False + return True + + def is_super(self) -> bool: + for tkn in self.body: + if tkn.kind == "IDENTIFIER" and tkn.text == "oparg1": + return True + return False + + +Part = Uop | Skip + + +@dataclass +class Instruction: + name: str + parts: list[Part] + _properties: Properties | None + is_target: bool = False + family: Optional["Family"] = None + opcode: int = -1 + + @property + def properties(self) -> Properties: + if self._properties is None: + self._properties = self._compute_properties() + return self._properties + + def _compute_properties(self) -> Properties: + return Properties.from_list([part.properties for part in self.parts]) + + def dump(self, indent: str) -> None: + print(indent, self.name, "=", ", ".join([part.name for part in self.parts])) + self.properties.dump(" " + indent) + + @property + def size(self) -> int: + return 1 + sum(part.size for part in self.parts) + + def is_super(self) -> bool: + if len(self.parts) != 1: + return False + uop = self.parts[0] + if isinstance(uop, Uop): + return uop.is_super() + else: + return False + + +@dataclass +class PseudoInstruction: + name: str + targets: list[Instruction] + flags: list[str] + opcode: int = -1 + + def dump(self, indent: str) -> None: + print(indent, self.name, "->", " or ".join([t.name for t in self.targets])) + + @property + def properties(self) -> Properties: + return Properties.from_list([i.properties for i in self.targets]) + + +@dataclass +class Family: + name: str + size: str + members: list[Instruction] + + def dump(self, indent: str) -> None: + print(indent, self.name, "= ", ", ".join([m.name for m in self.members])) + + +@dataclass +class Analysis: + instructions: dict[str, Instruction] + uops: dict[str, Uop] + families: dict[str, Family] + pseudos: dict[str, PseudoInstruction] + opmap: dict[str, int] + have_arg: int + min_instrumented: int + + +def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: + # To do -- support file and line output + # Construct a SyntaxError instance from message and token + return lexer.make_syntax_error(message, tkn.filename, tkn.line, tkn.column, "") + + +def override_error( + name: str, + context: parser.Context | None, + prev_context: parser.Context | None, + token: lexer.Token, +) -> SyntaxError: + return analysis_error( + f"Duplicate definition of '{name}' @ {context} " + f"previous definition @ {prev_context}", + token, + ) + + +def convert_stack_item(item: parser.StackEffect) -> StackItem: + return StackItem(item.name, item.type, item.cond, (item.size or "1")) + + +def analyze_stack(op: parser.InstDef) -> StackEffect: + inputs: list[StackItem] = [ + convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect) + ] + outputs: list[StackItem] = [convert_stack_item(i) for i in op.outputs] + for input, output in zip(inputs, outputs): + if input.name == output.name: + input.peek = output.peek = True + return StackEffect(inputs, outputs) + + +def analyze_caches(inputs: list[parser.InputEffect]) -> list[CacheEntry]: + caches: list[parser.CacheEffect] = [ + i for i in inputs if isinstance(i, parser.CacheEffect) + ] + for cache in caches: + if cache.name == "unused": + raise analysis_error( + "Unused cache entry in op. Move to enclosing macro.", cache.tokens[0] + ) + return [CacheEntry(i.name, int(i.size)) for i in caches] + + +def variable_used(node: parser.InstDef, name: str) -> bool: + """Determine whether a variable with a given name is used in a node.""" + return any( + token.kind == "IDENTIFIER" and token.text == name for token in node.tokens + ) + + +def is_infallible(op: parser.InstDef) -> bool: + return not ( + variable_used(op, "ERROR_IF") + or variable_used(op, "error") + or variable_used(op, "pop_1_error") + or variable_used(op, "exception_unwind") + or variable_used(op, "resume_with_error") + ) + + +NON_ESCAPING_FUNCTIONS = ( + "Py_INCREF", + "_PyDictOrValues_IsValues", + "_PyObject_DictOrValuesPointer", + "_PyDictOrValues_GetValues", + "_PyObject_MakeInstanceAttributesFromDict", + "Py_DECREF", + "_Py_DECREF_SPECIALIZED", + "DECREF_INPUTS_AND_REUSE_FLOAT", + "PyUnicode_Append", + "_PyLong_IsZero", + "Py_SIZE", + "Py_TYPE", + "PyList_GET_ITEM", + "PyTuple_GET_ITEM", + "PyList_GET_SIZE", + "PyTuple_GET_SIZE", + "Py_ARRAY_LENGTH", + "Py_Unicode_GET_LENGTH", + "PyUnicode_READ_CHAR", + "_Py_SINGLETON", + "PyUnicode_GET_LENGTH", + "_PyLong_IsCompact", + "_PyLong_IsNonNegativeCompact", + "_PyLong_CompactValue", + "_Py_NewRef", + "_Py_IsImmortal", + "_Py_STR", + "_PyLong_Add", + "_PyLong_Multiply", + "_PyLong_Subtract", + "Py_NewRef", + "_PyList_ITEMS", + "_PyTuple_ITEMS", + "_PyList_AppendTakeRef", + "_Py_atomic_load_uintptr_relaxed", + "_PyFrame_GetCode", + "_PyThreadState_HasStackSpace", +) + +ESCAPING_FUNCTIONS = ( + "import_name", + "import_from", +) + + +def makes_escaping_api_call(instr: parser.InstDef) -> bool: + if "CALL_INTRINSIC" in instr.name: + return True + tkns = iter(instr.tokens) + for tkn in tkns: + if tkn.kind != lexer.IDENTIFIER: + continue + try: + next_tkn = next(tkns) + except StopIteration: + return False + if next_tkn.kind != lexer.LPAREN: + continue + if tkn.text in ESCAPING_FUNCTIONS: + return True + if not tkn.text.startswith("Py") and not tkn.text.startswith("_Py"): + continue + if tkn.text.endswith("Check"): + continue + if tkn.text.startswith("Py_Is"): + continue + if tkn.text.endswith("CheckExact"): + continue + if tkn.text in NON_ESCAPING_FUNCTIONS: + continue + return True + return False + + + +EXITS = { + "DISPATCH", + "GO_TO_INSTRUCTION", + "Py_UNREACHABLE", + "DISPATCH_INLINED", + "DISPATCH_GOTO", +} + + +def eval_breaker_at_end(op: parser.InstDef) -> bool: + return op.tokens[-5].text == "CHECK_EVAL_BREAKER" + + +def always_exits(op: parser.InstDef) -> bool: + depth = 0 + tkn_iter = iter(op.tokens) + for tkn in tkn_iter: + if tkn.kind == "LBRACE": + depth += 1 + elif tkn.kind == "RBRACE": + depth -= 1 + elif depth > 1: + continue + elif tkn.kind == "GOTO" or tkn.kind == "RETURN": + return True + elif tkn.kind == "KEYWORD": + if tkn.text in EXITS: + return True + elif tkn.kind == "IDENTIFIER": + if tkn.text in EXITS: + return True + if tkn.text == "DEOPT_IF" or tkn.text == "ERROR_IF": + next(tkn_iter) # '(' + t = next(tkn_iter) + if t.text == "true": + return True + return False + + +def compute_properties(op: parser.InstDef) -> Properties: + has_free = ( + variable_used(op, "PyCell_New") + or variable_used(op, "PyCell_GET") + or variable_used(op, "PyCell_SET") + ) + return Properties( + escapes=makes_escaping_api_call(op), + infallible=is_infallible(op), + deopts=variable_used(op, "DEOPT_IF"), + oparg=variable_used(op, "oparg"), + jumps=variable_used(op, "JUMPBY"), + eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"), + ends_with_eval_breaker=eval_breaker_at_end(op), + needs_this=variable_used(op, "this_instr"), + always_exits=always_exits(op), + stores_sp=variable_used(op, "STORE_SP"), + tier_one_only=variable_used(op, "TIER_ONE_ONLY"), + uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), + uses_co_names=variable_used(op, "FRAME_CO_NAMES"), + uses_locals=(variable_used(op, "GETLOCAL") or variable_used(op, "SETLOCAL")) + and not has_free, + has_free=has_free, + ) + + +def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect]) -> Uop: + return Uop( + name=name, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=compute_properties(op), + ) + + +def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: + assert op.kind == "op" + if op.name in uops: + if "override" not in op.annotations: + raise override_error( + op.name, op.context, uops[op.name].context, op.tokens[0] + ) + uops[op.name] = make_uop(op.name, op, op.inputs) + + +def add_instruction( + name: str, parts: list[Part], instructions: dict[str, Instruction] +) -> None: + instructions[name] = Instruction(name, parts, None) + + +def desugar_inst( + inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: + assert inst.kind == "inst" + name = inst.name + op_inputs: list[parser.InputEffect] = [] + parts: list[Part] = [] + uop_index = -1 + # Move unused cache entries to the Instruction, removing them from the Uop. + for input in inst.inputs: + if isinstance(input, parser.CacheEffect) and input.name == "unused": + parts.append(Skip(input.size)) + else: + op_inputs.append(input) + if uop_index < 0: + uop_index = len(parts) + # Place holder for the uop. + parts.append(Skip(0)) + uop = make_uop("_" + inst.name, inst, op_inputs) + uop.implicitly_created = True + uops[inst.name] = uop + if uop_index < 0: + parts.append(uop) + else: + parts[uop_index] = uop + add_instruction(name, parts, instructions) + + +def add_macro( + macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: + parts: list[Uop | Skip] = [] + for part in macro.uops: + match part: + case parser.OpName(): + if part.name not in uops: + analysis_error(f"No Uop named {part.name}", macro.tokens[0]) + parts.append(uops[part.name]) + case parser.CacheEffect(): + parts.append(Skip(part.size)) + case _: + assert False + assert parts + add_instruction(macro.name, parts, instructions) + + +def add_family( + pfamily: parser.Family, + instructions: dict[str, Instruction], + families: dict[str, Family], +) -> None: + family = Family( + pfamily.name, + pfamily.size, + [instructions[member_name] for member_name in pfamily.members], + ) + for member in family.members: + member.family = family + # The head of the family is an implicit jump target for DEOPTs + instructions[family.name].is_target = True + families[family.name] = family + + +def add_pseudo( + pseudo: parser.Pseudo, + instructions: dict[str, Instruction], + pseudos: dict[str, PseudoInstruction], +) -> None: + pseudos[pseudo.name] = PseudoInstruction( + pseudo.name, + [instructions[target] for target in pseudo.targets], + pseudo.flags, + ) + + +def assign_opcodes( + instructions: dict[str, Instruction], + families: dict[str, Family], + pseudos: dict[str, PseudoInstruction], +) -> tuple[dict[str, int], int, int]: + """Assigns opcodes, then returns the opmap, + have_arg and min_instrumented values""" + instmap: dict[str, int] = {} + + # 0 is reserved for cache entries. This helps debugging. + instmap["CACHE"] = 0 + + # 17 is reserved as it is the initial value for the specializing counter. + # This helps catch cases where we attempt to execute a cache. + instmap["RESERVED"] = 17 + + # 149 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py + instmap["RESUME"] = 149 + + # This is an historical oddity. + instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 + + instmap["INSTRUMENTED_LINE"] = 254 + + instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] + + # Special case: this instruction is implemented in ceval.c + # rather than bytecodes.c, so we need to add it explicitly + # here (at least until we add something to bytecodes.c to + # declare external instructions). + instrumented.append("INSTRUMENTED_LINE") + + specialized: set[str] = set() + no_arg: list[str] = [] + has_arg: list[str] = [] + + for family in families.values(): + specialized.update(inst.name for inst in family.members) + + for inst in instructions.values(): + name = inst.name + if name in specialized: + continue + if name in instrumented: + continue + if inst.properties.oparg: + has_arg.append(name) + else: + no_arg.append(name) + + # Specialized ops appear in their own section + # Instrumented opcodes are at the end of the valid range + min_internal = 150 + min_instrumented = 254 - (len(instrumented) - 1) + assert min_internal + len(specialized) < min_instrumented + + next_opcode = 1 + + def add_instruction(name: str) -> None: + nonlocal next_opcode + if name in instmap: + return # Pre-defined name + while next_opcode in instmap.values(): + next_opcode += 1 + instmap[name] = next_opcode + next_opcode += 1 + + for name in sorted(no_arg): + add_instruction(name) + for name in sorted(has_arg): + add_instruction(name) + # For compatibility + next_opcode = min_internal + for name in sorted(specialized): + add_instruction(name) + next_opcode = min_instrumented + for name in instrumented: + add_instruction(name) + + for name in instructions: + instructions[name].opcode = instmap[name] + + for op, name in enumerate(sorted(pseudos), 256): + instmap[name] = op + pseudos[name].opcode = op + + return instmap, len(no_arg), min_instrumented + + +def analyze_forest(forest: list[parser.AstNode]) -> Analysis: + instructions: dict[str, Instruction] = {} + uops: dict[str, Uop] = {} + families: dict[str, Family] = {} + pseudos: dict[str, PseudoInstruction] = {} + for node in forest: + match node: + case parser.InstDef(name): + if node.kind == "inst": + desugar_inst(node, instructions, uops) + else: + assert node.kind == "op" + add_op(node, uops) + case parser.Macro(): + pass + case parser.Family(): + pass + case parser.Pseudo(): + pass + case _: + assert False + for node in forest: + if isinstance(node, parser.Macro): + add_macro(node, instructions, uops) + for node in forest: + match node: + case parser.Family(): + add_family(node, instructions, families) + case parser.Pseudo(): + add_pseudo(node, instructions, pseudos) + case _: + pass + for uop in uops.values(): + tkn_iter = iter(uop.body) + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text == "GO_TO_INSTRUCTION": + if next(tkn_iter).kind != "LPAREN": + continue + target = next(tkn_iter) + if target.kind != "IDENTIFIER": + continue + if target.text in instructions: + instructions[target.text].is_target = True + # Special case BINARY_OP_INPLACE_ADD_UNICODE + # BINARY_OP_INPLACE_ADD_UNICODE is not a normal family member, + # as it is the wrong size, but we need it to maintain an + # historical optimization. + if "BINARY_OP_INPLACE_ADD_UNICODE" in instructions: + inst = instructions["BINARY_OP_INPLACE_ADD_UNICODE"] + inst.family = families["BINARY_OP"] + families["BINARY_OP"].members.append(inst) + opmap, first_arg, min_instrumented = assign_opcodes( + instructions, families, pseudos + ) + return Analysis( + instructions, uops, families, pseudos, opmap, first_arg, min_instrumented + ) + + +def analyze_files(filenames: list[str]) -> Analysis: + return analyze_forest(parser.parse_files(filenames)) + + +def dump_analysis(analysis: Analysis) -> None: + print("Uops:") + for u in analysis.uops.values(): + u.dump(" ") + print("Instructions:") + for i in analysis.instructions.values(): + i.dump(" ") + print("Families:") + for f in analysis.families.values(): + f.dump(" ") + print("Pseudos:") + for p in analysis.pseudos.values(): + p.dump(" ") + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("No input") + else: + filenames = sys.argv[1:] + dump_analysis(analyze_files(filenames)) diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py new file mode 100644 index 00000000000000..069f0177a74018 --- /dev/null +++ b/Tools/cases_generator/cwriter.py @@ -0,0 +1,146 @@ +import contextlib +from lexer import Token +from typing import TextIO, Iterator + + +class CWriter: + "A writer that understands tokens and how to format C code" + + last_token: Token | None + + def __init__(self, out: TextIO, indent: int, line_directives: bool): + self.out = out + self.base_column = indent * 4 + self.indents = [i * 4 for i in range(indent + 1)] + self.line_directives = line_directives + self.last_token = None + self.newline = True + + def set_position(self, tkn: Token) -> None: + if self.last_token is not None: + if self.last_token.line < tkn.line: + self.out.write("\n") + if self.line_directives: + self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') + self.out.write(" " * self.indents[-1]) + else: + gap = tkn.column - self.last_token.end_column + self.out.write(" " * gap) + elif self.newline: + self.out.write(" " * self.indents[-1]) + self.last_token = tkn + self.newline = False + + def emit_at(self, txt: str, where: Token) -> None: + self.set_position(where) + self.out.write(txt) + + def maybe_dedent(self, txt: str) -> None: + parens = txt.count("(") - txt.count(")") + if parens < 0: + self.indents.pop() + braces = txt.count("{") - txt.count("}") + if braces < 0 or is_label(txt): + self.indents.pop() + + def maybe_indent(self, txt: str) -> None: + parens = txt.count("(") - txt.count(")") + if parens > 0: + if self.last_token: + offset = self.last_token.end_column - 1 + if offset <= self.indents[-1] or offset > 40: + offset = self.indents[-1] + 4 + else: + offset = self.indents[-1] + 4 + self.indents.append(offset) + if is_label(txt): + self.indents.append(self.indents[-1] + 4) + else: + braces = txt.count("{") - txt.count("}") + if braces > 0: + assert braces == 1 + if 'extern "C"' in txt: + self.indents.append(self.indents[-1]) + else: + self.indents.append(self.indents[-1] + 4) + + def emit_text(self, txt: str) -> None: + self.out.write(txt) + + def emit_multiline_comment(self, tkn: Token) -> None: + self.set_position(tkn) + lines = tkn.text.splitlines(True) + first = True + for line in lines: + text = line.lstrip() + if first: + spaces = 0 + else: + spaces = self.indents[-1] + if text.startswith("*"): + spaces += 1 + else: + spaces += 3 + first = False + self.out.write(" " * spaces) + self.out.write(text) + + def emit_token(self, tkn: Token) -> None: + if tkn.kind == "COMMENT" and "\n" in tkn.text: + return self.emit_multiline_comment(tkn) + self.maybe_dedent(tkn.text) + self.set_position(tkn) + self.emit_text(tkn.text) + self.maybe_indent(tkn.text) + + def emit_str(self, txt: str) -> None: + self.maybe_dedent(txt) + if self.newline and txt: + if txt[0] != "\n": + self.out.write(" " * self.indents[-1]) + self.newline = False + self.emit_text(txt) + if txt.endswith("\n"): + self.newline = True + self.maybe_indent(txt) + self.last_token = None + + def emit(self, txt: str | Token) -> None: + if isinstance(txt, Token): + self.emit_token(txt) + elif isinstance(txt, str): + self.emit_str(txt) + else: + assert False + + def start_line(self) -> None: + if not self.newline: + self.out.write("\n") + self.newline = True + self.last_token = None + + @contextlib.contextmanager + def header_guard(self, name: str) -> Iterator[None]: + self.out.write( + f""" +#ifndef {name} +#define {name} +#ifdef __cplusplus +extern "C" {{ +#endif + +""" + ) + yield + self.out.write( + f""" +#ifdef __cplusplus +}} +#endif +#endif /* !{name} */ +""" + ) + + +def is_label(txt: str) -> bool: + return not txt.startswith("//") and txt.endswith(":") diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py deleted file mode 100644 index bf76112159e38e..00000000000000 --- a/Tools/cases_generator/flags.py +++ /dev/null @@ -1,191 +0,0 @@ -import dataclasses - -from formatting import Formatter -import lexer as lx -import parsing -from typing import AbstractSet - -NON_ESCAPING_FUNCTIONS = ( - "Py_INCREF", - "_PyDictOrValues_IsValues", - "_PyObject_DictOrValuesPointer", - "_PyDictOrValues_GetValues", - "_PyObject_MakeInstanceAttributesFromDict", - "Py_DECREF", - "_Py_DECREF_SPECIALIZED", - "DECREF_INPUTS_AND_REUSE_FLOAT", - "PyUnicode_Append", - "_PyLong_IsZero", - "Py_SIZE", - "Py_TYPE", - "PyList_GET_ITEM", - "PyTuple_GET_ITEM", - "PyList_GET_SIZE", - "PyTuple_GET_SIZE", - "Py_ARRAY_LENGTH", - "Py_Unicode_GET_LENGTH", - "PyUnicode_READ_CHAR", - "_Py_SINGLETON", - "PyUnicode_GET_LENGTH", - "_PyLong_IsCompact", - "_PyLong_IsNonNegativeCompact", - "_PyLong_CompactValue", - "_Py_NewRef", - "_Py_IsImmortal", - "_Py_STR", - "_PyLong_Add", - "_PyLong_Multiply", - "_PyLong_Subtract", - "Py_NewRef", - "_PyList_ITEMS", - "_PyTuple_ITEMS", - "_PyList_AppendTakeRef", - "_Py_atomic_load_uintptr_relaxed", - "_PyFrame_GetCode", - "_PyThreadState_HasStackSpace", -) - -ESCAPING_FUNCTIONS = ( - "import_name", - "import_from", -) - - -def makes_escaping_api_call(instr: parsing.InstDef) -> bool: - if "CALL_INTRINSIC" in instr.name: - return True - tkns = iter(instr.tokens) - for tkn in tkns: - if tkn.kind != lx.IDENTIFIER: - continue - try: - next_tkn = next(tkns) - except StopIteration: - return False - if next_tkn.kind != lx.LPAREN: - continue - if tkn.text in ESCAPING_FUNCTIONS: - return True - if not tkn.text.startswith("Py") and not tkn.text.startswith("_Py"): - continue - if tkn.text.endswith("Check"): - continue - if tkn.text.startswith("Py_Is"): - continue - if tkn.text.endswith("CheckExact"): - continue - if tkn.text in NON_ESCAPING_FUNCTIONS: - continue - return True - return False - - -@dataclasses.dataclass -class InstructionFlags: - """Construct and manipulate instruction flags""" - - HAS_ARG_FLAG: bool = False - HAS_CONST_FLAG: bool = False - HAS_NAME_FLAG: bool = False - HAS_JUMP_FLAG: bool = False - HAS_FREE_FLAG: bool = False - HAS_LOCAL_FLAG: bool = False - HAS_EVAL_BREAK_FLAG: bool = False - HAS_DEOPT_FLAG: bool = False - HAS_ERROR_FLAG: bool = False - HAS_ESCAPES_FLAG: bool = False - - def __post_init__(self) -> None: - self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} - - @staticmethod - def fromInstruction(instr: parsing.InstDef) -> "InstructionFlags": - has_free = ( - variable_used(instr, "PyCell_New") - or variable_used(instr, "PyCell_GET") - or variable_used(instr, "PyCell_SET") - ) - - return InstructionFlags( - HAS_ARG_FLAG=variable_used(instr, "oparg"), - HAS_CONST_FLAG=variable_used(instr, "FRAME_CO_CONSTS"), - HAS_NAME_FLAG=variable_used(instr, "FRAME_CO_NAMES"), - HAS_JUMP_FLAG=variable_used(instr, "JUMPBY"), - HAS_FREE_FLAG=has_free, - HAS_LOCAL_FLAG=( - variable_used(instr, "GETLOCAL") or variable_used(instr, "SETLOCAL") - ) - and not has_free, - HAS_EVAL_BREAK_FLAG=variable_used(instr, "CHECK_EVAL_BREAKER"), - HAS_DEOPT_FLAG=variable_used(instr, "DEOPT_IF"), - HAS_ERROR_FLAG=( - variable_used(instr, "ERROR_IF") - or variable_used(instr, "error") - or variable_used(instr, "pop_1_error") - or variable_used(instr, "exception_unwind") - or variable_used(instr, "resume_with_error") - ), - HAS_ESCAPES_FLAG=makes_escaping_api_call(instr), - ) - - @staticmethod - def newEmpty() -> "InstructionFlags": - return InstructionFlags() - - def add(self, other: "InstructionFlags") -> None: - for name, value in dataclasses.asdict(other).items(): - if value: - setattr(self, name, value) - - def names(self, value: bool | None = None) -> list[str]: - if value is None: - return list(dataclasses.asdict(self).keys()) - return [n for n, v in dataclasses.asdict(self).items() if v == value] - - def bitmap(self, ignore: AbstractSet[str] = frozenset()) -> int: - flags = 0 - assert all(hasattr(self, name) for name in ignore) - for name in self.names(): - if getattr(self, name) and name not in ignore: - flags |= self.bitmask[name] - return flags - - @classmethod - def emit_macros(cls, out: Formatter) -> None: - flags = cls.newEmpty() - for name, value in flags.bitmask.items(): - out.emit(f"#define {name} ({value})") - - for name, value in flags.bitmask.items(): - out.emit( - f"#define OPCODE_{name[:-len('_FLAG')]}(OP) " - f"(_PyOpcode_opcode_metadata[OP].flags & ({name}))" - ) - - -def variable_used(node: parsing.Node, name: str) -> bool: - """Determine whether a variable with a given name is used in a node.""" - return any( - token.kind == "IDENTIFIER" and token.text == name for token in node.tokens - ) - - -def variable_used_unspecialized(node: parsing.Node, name: str) -> bool: - """Like variable_used(), but skips #if ENABLE_SPECIALIZATION blocks.""" - tokens: list[lx.Token] = [] - skipping = False - for i, token in enumerate(node.tokens): - if token.kind == "CMACRO": - text = "".join(token.text.split()) - # TODO: Handle nested #if - if text == "#if": - if i + 1 < len(node.tokens) and node.tokens[i + 1].text in ( - "ENABLE_SPECIALIZATION", - "TIER_ONE", - ): - skipping = True - elif text in ("#else", "#endif"): - skipping = False - if not skipping: - tokens.append(token) - return any(token.kind == "IDENTIFIER" and token.text == name for token in tokens) diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py deleted file mode 100644 index 4fd9172d20c274..00000000000000 --- a/Tools/cases_generator/formatting.py +++ /dev/null @@ -1,206 +0,0 @@ -import contextlib -import re -import typing -from collections.abc import Iterator - -from parsing import StackEffect, Family - -UNUSED = "unused" - - -class Formatter: - """Wraps an output stream with the ability to indent etc.""" - - stream: typing.TextIO - prefix: str - emit_line_directives: bool = False - lineno: int # Next line number, 1-based - filename: str # Slightly improved stream.filename - nominal_lineno: int - nominal_filename: str - - def __init__( - self, - stream: typing.TextIO, - indent: int, - emit_line_directives: bool = False, - comment: str = "//", - ) -> None: - self.stream = stream - self.prefix = " " * indent - self.emit_line_directives = emit_line_directives - self.comment = comment - self.lineno = 1 - self.filename = prettify_filename(self.stream.name) - self.nominal_lineno = 1 - self.nominal_filename = self.filename - - def write_raw(self, s: str) -> None: - self.stream.write(s) - newlines = s.count("\n") - self.lineno += newlines - self.nominal_lineno += newlines - - def emit(self, arg: str) -> None: - if arg: - self.write_raw(f"{self.prefix}{arg}\n") - else: - self.write_raw("\n") - - def set_lineno(self, lineno: int, filename: str) -> None: - if self.emit_line_directives: - if lineno != self.nominal_lineno or filename != self.nominal_filename: - self.emit(f'#line {lineno} "{filename}"') - self.nominal_lineno = lineno - self.nominal_filename = filename - - def reset_lineno(self) -> None: - if self.lineno != self.nominal_lineno or self.filename != self.nominal_filename: - self.set_lineno(self.lineno + 1, self.filename) - - @contextlib.contextmanager - def indent(self) -> Iterator[None]: - self.prefix += " " - yield - self.prefix = self.prefix[:-4] - - @contextlib.contextmanager - def block(self, head: str, tail: str = "") -> Iterator[None]: - if head: - self.emit(head + " {") - else: - self.emit("{") - with self.indent(): - yield - self.emit("}" + tail) - - def stack_adjust( - self, - input_effects: list[StackEffect], - output_effects: list[StackEffect], - ) -> None: - shrink, isym = list_effect_size(input_effects) - grow, osym = list_effect_size(output_effects) - diff = grow - shrink - if isym and isym != osym: - self.emit(f"STACK_SHRINK({isym});") - if diff < 0: - self.emit(f"STACK_SHRINK({-diff});") - if diff > 0: - self.emit(f"STACK_GROW({diff});") - if osym and osym != isym: - self.emit(f"STACK_GROW({osym});") - - def declare(self, dst: StackEffect, src: StackEffect | None) -> None: - if dst.name == UNUSED or dst.cond == "0": - return - typ = f"{dst.type}" if dst.type else "PyObject *" - if src: - cast = self.cast(dst, src) - initexpr = f"{cast}{src.name}" - if src.cond and src.cond != "1": - initexpr = f"{parenthesize_cond(src.cond)} ? {initexpr} : NULL" - init = f" = {initexpr}" - elif dst.cond and dst.cond != "1": - init = " = NULL" - else: - init = "" - sepa = "" if typ.endswith("*") else " " - self.emit(f"{typ}{sepa}{dst.name}{init};") - - def assign(self, dst: StackEffect, src: StackEffect) -> None: - if src.name == UNUSED or dst.name == UNUSED: - return - cast = self.cast(dst, src) - if re.match(r"^REG\(oparg(\d+)\)$", dst.name): - self.emit(f"Py_XSETREF({dst.name}, {cast}{src.name});") - else: - stmt = f"{dst.name} = {cast}{src.name};" - if src.cond and src.cond != "1": - if src.cond == "0": - # It will not be executed - return - stmt = f"if ({src.cond}) {{ {stmt} }}" - self.emit(stmt) - - def cast(self, dst: StackEffect, src: StackEffect) -> str: - return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" - - def static_assert_family_size( - self, name: str, family: Family | None, cache_offset: int - ) -> None: - """Emit a static_assert for the size of a family, if known. - - This will fail at compile time if the cache size computed from - the instruction definition does not match the size of the struct - used by specialize.c. - """ - if family and name == family.name: - cache_size = family.size - if cache_size: - self.emit( - f"static_assert({cache_size} == {cache_offset}, " - f'"incorrect cache size");' - ) - - -def prettify_filename(filename: str) -> str: - # Make filename more user-friendly and less platform-specific, - # it is only used for error reporting at this point. - filename = filename.replace("\\", "/") - if filename.startswith("./"): - filename = filename[2:] - if filename.endswith(".new"): - filename = filename[:-4] - return filename - - -def list_effect_size(effects: list[StackEffect]) -> tuple[int, str]: - numeric = 0 - symbolic: list[str] = [] - for effect in effects: - diff, sym = effect_size(effect) - numeric += diff - if sym: - symbolic.append(maybe_parenthesize(sym)) - return numeric, " + ".join(symbolic) - - -def effect_size(effect: StackEffect) -> tuple[int, str]: - """Return the 'size' impact of a stack effect. - - Returns a tuple (numeric, symbolic) where: - - - numeric is an int giving the statically analyzable size of the effect - - symbolic is a string representing a variable effect (e.g. 'oparg*2') - - At most one of these will be non-zero / non-empty. - """ - if effect.size: - assert not effect.cond, "Array effects cannot have a condition" - return 0, effect.size - elif effect.cond: - if effect.cond in ("0", "1"): - return int(effect.cond), "" - return 0, f"{maybe_parenthesize(effect.cond)} ? 1 : 0" - else: - return 1, "" - - -def maybe_parenthesize(sym: str) -> str: - """Add parentheses around a string if it contains an operator. - - An exception is made for '*' which is common and harmless - in the context where the symbolic size is used. - """ - if re.match(r"^[\s\w*]+$", sym): - return sym - else: - return f"({sym})" - - -def parenthesize_cond(cond: str) -> str: - """Parenthesize a condition, but only if it contains ?: itself.""" - if "?" in cond: - cond = f"({cond})" - return cond diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py deleted file mode 100644 index 1f94c1fedb2ac7..00000000000000 --- a/Tools/cases_generator/generate_cases.py +++ /dev/null @@ -1,898 +0,0 @@ -"""Generate the main interpreter switch. -Reads the instruction definitions from bytecodes.c. -Writes the cases to generated_cases.c.h, which is #included in ceval.c. -""" - -import argparse -import contextlib -import itertools -import os -import posixpath -import sys -import textwrap -import typing -from collections.abc import Iterator - -import stacking # Early import to avoid circular import -from _typing_backports import assert_never -from analysis import Analyzer -from formatting import Formatter, list_effect_size -from flags import InstructionFlags, variable_used -from instructions import ( - AnyInstruction, - AbstractInstruction, - Component, - Instruction, - MacroInstruction, - MacroParts, - PseudoInstruction, - TIER_ONE, - TIER_TWO, -) -import parsing -from parsing import StackEffect - - -HERE = os.path.dirname(__file__) -ROOT = os.path.join(HERE, "../..") -THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, posixpath.sep) - -DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) -DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h")) -DEFAULT_OPCODE_IDS_H_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Include/opcode_ids.h") -) -DEFAULT_OPCODE_TARGETS_H_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Python/opcode_targets.h") -) -DEFAULT_METADATA_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Include/internal/pycore_opcode_metadata.h") -) -DEFAULT_PYMETADATA_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Lib/_opcode_metadata.py") -) -DEFAULT_EXECUTOR_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Python/executor_cases.c.h") -) -DEFAULT_ABSTRACT_INTERPRETER_OUTPUT = os.path.relpath( - os.path.join(ROOT, "Python/abstract_interp_cases.c.h") -) - -# Constants used instead of size for macro expansions. -# Note: 1, 2, 4 must match actual cache entry sizes. -OPARG_SIZES = { - "OPARG_FULL": 0, - "OPARG_CACHE_1": 1, - "OPARG_CACHE_2": 2, - "OPARG_CACHE_4": 4, - "OPARG_TOP": 5, - "OPARG_BOTTOM": 6, - "OPARG_SAVE_RETURN_OFFSET": 7, -} - -INSTR_FMT_PREFIX = "INSTR_FMT_" - -# TODO: generate all these after updating the DSL -SPECIALLY_HANDLED_ABSTRACT_INSTR = { - "LOAD_FAST", - "LOAD_FAST_CHECK", - "LOAD_FAST_AND_CLEAR", - "LOAD_CONST", - "STORE_FAST", - "STORE_FAST_MAYBE_NULL", - "COPY", - # Arithmetic - "_BINARY_OP_MULTIPLY_INT", - "_BINARY_OP_ADD_INT", - "_BINARY_OP_SUBTRACT_INT", -} - -arg_parser = argparse.ArgumentParser( - description="Generate the code for the interpreter switch.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, -) - -arg_parser.add_argument( - "-v", - "--viable", - help="Print list of non-viable uops and exit", - action="store_true", -) -arg_parser.add_argument( - "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT -) -arg_parser.add_argument( - "-n", - "--opcode_ids_h", - type=str, - help="Header file with opcode number definitions", - default=DEFAULT_OPCODE_IDS_H_OUTPUT, -) -arg_parser.add_argument( - "-t", - "--opcode_targets_h", - type=str, - help="File with opcode targets for computed gotos", - default=DEFAULT_OPCODE_TARGETS_H_OUTPUT, -) -arg_parser.add_argument( - "-m", - "--metadata", - type=str, - help="Generated C metadata", - default=DEFAULT_METADATA_OUTPUT, -) -arg_parser.add_argument( - "-p", - "--pymetadata", - type=str, - help="Generated Python metadata", - default=DEFAULT_PYMETADATA_OUTPUT, -) -arg_parser.add_argument( - "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" -) -arg_parser.add_argument( - "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" -) -arg_parser.add_argument( - "-e", - "--executor-cases", - type=str, - help="Write executor cases to this file", - default=DEFAULT_EXECUTOR_OUTPUT, -) -arg_parser.add_argument( - "-a", - "--abstract-interpreter-cases", - type=str, - help="Write abstract interpreter cases to this file", - default=DEFAULT_ABSTRACT_INTERPRETER_OUTPUT, -) - - -class Generator(Analyzer): - def get_stack_effect_info( - self, thing: parsing.InstDef | parsing.Macro | parsing.Pseudo - ) -> tuple[AnyInstruction | None, str, str]: - def effect_str(effects: list[StackEffect]) -> str: - n_effect, sym_effect = list_effect_size(effects) - if sym_effect: - return f"{sym_effect} + {n_effect}" if n_effect else sym_effect - return str(n_effect) - - instr: AnyInstruction | None - popped: str | None = None - pushed: str | None = None - match thing: - case parsing.InstDef(): - instr = self.instrs[thing.name] - popped = effect_str(instr.input_effects) - pushed = effect_str(instr.output_effects) - case parsing.Macro(): - instr = self.macro_instrs[thing.name] - popped, pushed = stacking.get_stack_effect_info_for_macro(instr) - case parsing.Pseudo(): - instr = self.pseudo_instrs[thing.name] - # Calculate stack effect, and check that it's the same - # for all targets. - for target in self.pseudos[thing.name].targets: - target_instr = self.instrs.get(target) - if target_instr is None: - macro_instr = self.macro_instrs[target] - popped, pushed = stacking.get_stack_effect_info_for_macro(macro_instr) - else: - target_popped = effect_str(target_instr.input_effects) - target_pushed = effect_str(target_instr.output_effects) - if popped is None: - popped, pushed = target_popped, target_pushed - else: - assert popped == target_popped - assert pushed == target_pushed - case _: - assert_never(thing) - assert popped is not None and pushed is not None - return instr, popped, pushed - - @contextlib.contextmanager - def metadata_item(self, signature: str, open: str, close: str) -> Iterator[None]: - self.out.emit("") - self.out.emit(f"extern {signature};") - self.out.emit("#ifdef NEED_OPCODE_METADATA") - with self.out.block(f"{signature} {open}", close): - yield - self.out.emit("#endif // NEED_OPCODE_METADATA") - - def write_stack_effect_functions(self) -> None: - popped_data: list[tuple[AnyInstruction, str]] = [] - pushed_data: list[tuple[AnyInstruction, str]] = [] - for thing in self.everything: - if isinstance(thing, parsing.Macro) and thing.name in self.instrs: - continue - instr, popped, pushed = self.get_stack_effect_info(thing) - if instr is not None: - popped_data.append((instr, popped)) - pushed_data.append((instr, pushed)) - - def write_function( - direction: str, data: list[tuple[AnyInstruction, str]] - ) -> None: - with self.metadata_item( - f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", - "", - "", - ): - with self.out.block("switch(opcode)"): - for instr, effect in data: - self.out.emit(f"case {instr.name}:") - self.out.emit(f" return {effect};") - self.out.emit("default:") - self.out.emit(" return -1;") - - write_function("popped", popped_data) - write_function("pushed", pushed_data) - self.out.emit("") - - def from_source_files(self) -> str: - filenames = [] - for filename in self.input_filenames: - try: - filename = os.path.relpath(filename, ROOT) - except ValueError: - # May happen on Windows if root and temp on different volumes - pass - filenames.append(filename.replace(os.path.sep, posixpath.sep)) - paths = f"\n{self.out.comment} ".join(filenames) - return f"{self.out.comment} from:\n{self.out.comment} {paths}\n" - - def write_provenance_header(self) -> None: - self.out.write_raw(f"{self.out.comment} This file is generated by {THIS}\n") - self.out.write_raw(self.from_source_files()) - self.out.write_raw(f"{self.out.comment} Do not edit!\n") - - def assign_opcode_ids(self) -> None: - """Assign IDs to opcodes""" - - ops: list[tuple[bool, str]] = [] # (has_arg, name) for each opcode - instrumented_ops: list[str] = [] - - specialized_ops: set[str] = set() - for name, family in self.families.items(): - specialized_ops.update(family.members) - - for instr in self.macro_instrs.values(): - name = instr.name - if name in specialized_ops: - continue - if name.startswith("INSTRUMENTED_"): - instrumented_ops.append(name) - else: - ops.append((instr.instr_flags.HAS_ARG_FLAG, name)) - - # Special case: this instruction is implemented in ceval.c - # rather than bytecodes.c, so we need to add it explicitly - # here (at least until we add something to bytecodes.c to - # declare external instructions). - instrumented_ops.append("INSTRUMENTED_LINE") - - # assert lists are unique - assert len(set(ops)) == len(ops) - assert len(set(instrumented_ops)) == len(instrumented_ops) - - opname: list[str | None] = [None] * 512 - opmap: dict[str, int] = {} - markers: dict[str, int] = {} - - def map_op(op: int, name: str) -> None: - assert op < len(opname) - assert opname[op] is None, (op, name) - assert name not in opmap - opname[op] = name - opmap[name] = op - - # 0 is reserved for cache entries. This helps debugging. - map_op(0, "CACHE") - - # 17 is reserved as it is the initial value for the specializing counter. - # This helps catch cases where we attempt to execute a cache. - map_op(17, "RESERVED") - - # 149 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py - map_op(149, "RESUME") - - # Specialized ops appear in their own section - # Instrumented opcodes are at the end of the valid range - min_internal = 150 - min_instrumented = 254 - (len(instrumented_ops) - 1) - assert min_internal + len(specialized_ops) < min_instrumented - - next_opcode = 1 - for has_arg, name in sorted(ops): - if name in opmap: - continue # an anchored name, like CACHE - map_op(next_opcode, name) - if has_arg and "HAVE_ARGUMENT" not in markers: - markers["HAVE_ARGUMENT"] = next_opcode - - while opname[next_opcode] is not None: - next_opcode += 1 - - assert next_opcode < min_internal, next_opcode - - for i, op in enumerate(sorted(specialized_ops)): - map_op(min_internal + i, op) - - markers["MIN_INSTRUMENTED_OPCODE"] = min_instrumented - for i, op in enumerate(instrumented_ops): - map_op(min_instrumented + i, op) - - # Pseudo opcodes are after the valid range - for i, op in enumerate(sorted(self.pseudos)): - map_op(256 + i, op) - - assert 255 not in opmap.values() # 255 is reserved - self.opmap = opmap - self.markers = markers - - def write_opcode_ids( - self, opcode_ids_h_filename: str, opcode_targets_filename: str - ) -> None: - """Write header file that defined the opcode IDs""" - - with open(opcode_ids_h_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 0) - - self.write_provenance_header() - - self.out.emit("") - self.out.emit("#ifndef Py_OPCODE_IDS_H") - self.out.emit("#define Py_OPCODE_IDS_H") - self.out.emit("#ifdef __cplusplus") - self.out.emit('extern "C" {') - self.out.emit("#endif") - self.out.emit("") - self.out.emit("/* Instruction opcodes for compiled code */") - - def define(name: str, opcode: int) -> None: - self.out.emit(f"#define {name:<38} {opcode:>3}") - - all_pairs: list[tuple[int, int, str]] = [] - # the second item in the tuple sorts the markers before the ops - all_pairs.extend((i, 1, name) for (name, i) in self.markers.items()) - all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items()) - for i, _, name in sorted(all_pairs): - assert name is not None - define(name, i) - - self.out.emit("") - self.out.emit("#ifdef __cplusplus") - self.out.emit("}") - self.out.emit("#endif") - self.out.emit("#endif /* !Py_OPCODE_IDS_H */") - - with open(opcode_targets_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 0) - - with self.out.block("static void *opcode_targets[256] =", ";"): - targets = ["_unknown_opcode"] * 256 - for name, op in self.opmap.items(): - if op < 256: - targets[op] = f"TARGET_{name}" - f.write(",\n".join([f" &&{s}" for s in targets])) - - def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> None: - """Write instruction metadata to output file.""" - - # Compute the set of all instruction formats. - all_formats: set[str] = set() - for thing in self.everything: - format: str | None = None - match thing: - case parsing.InstDef(): - format = self.instrs[thing.name].instr_fmt - case parsing.Macro(): - format = self.macro_instrs[thing.name].instr_fmt - case parsing.Pseudo(): - # Pseudo instructions exist only in the compiler, - # so do not have a format - continue - case _: - assert_never(thing) - assert format is not None - all_formats.add(format) - - # Turn it into a sorted list of enum values. - format_enums = [INSTR_FMT_PREFIX + format for format in sorted(all_formats)] - - with open(metadata_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 0) - - self.write_provenance_header() - - self.out.emit("") - self.out.emit("#ifndef Py_BUILD_CORE") - self.out.emit('# error "this header requires Py_BUILD_CORE define"') - self.out.emit("#endif") - self.out.emit("") - self.out.emit("#include // bool") - - self.write_pseudo_instrs() - - self.out.emit("") - self.write_uop_items(lambda name, counter: f"#define {name} {counter}") - - self.write_stack_effect_functions() - - # Write the enum definition for instruction formats. - with self.out.block("enum InstructionFormat", ";"): - for enum in format_enums: - self.out.emit(enum + ",") - - self.out.emit("") - self.out.emit( - "#define IS_VALID_OPCODE(OP) \\\n" - " (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \\\n" - " (_PyOpcode_opcode_metadata[(OP)].valid_entry))" - ) - - self.out.emit("") - InstructionFlags.emit_macros(self.out) - - self.out.emit("") - with self.out.block("struct opcode_metadata", ";"): - self.out.emit("bool valid_entry;") - self.out.emit("enum InstructionFormat instr_format;") - self.out.emit("int flags;") - self.out.emit("") - - with self.out.block("struct opcode_macro_expansion", ";"): - self.out.emit("int nuops;") - self.out.emit( - "struct { int16_t uop; int8_t size; int8_t offset; } uops[12];" - ) - self.out.emit("") - - for key, value in OPARG_SIZES.items(): - self.out.emit(f"#define {key} {value}") - self.out.emit("") - - self.out.emit( - "#define OPCODE_METADATA_FLAGS(OP) " - "(_PyOpcode_opcode_metadata[(OP)].flags & (HAS_ARG_FLAG | HAS_JUMP_FLAG))" - ) - self.out.emit("#define SAME_OPCODE_METADATA(OP1, OP2) \\") - self.out.emit( - " (OPCODE_METADATA_FLAGS(OP1) == OPCODE_METADATA_FLAGS(OP2))" - ) - self.out.emit("") - - # Write metadata array declaration - self.out.emit("#define OPCODE_METADATA_SIZE 512") - self.out.emit("#define OPCODE_UOP_NAME_SIZE 512") - self.out.emit("#define OPCODE_MACRO_EXPANSION_SIZE 256") - - with self.metadata_item( - "const struct opcode_metadata " - "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]", - "=", - ";", - ): - # Write metadata for each instruction - for thing in self.everything: - match thing: - case parsing.InstDef(): - self.write_metadata_for_inst(self.instrs[thing.name]) - case parsing.Macro(): - if thing.name not in self.instrs: - self.write_metadata_for_macro( - self.macro_instrs[thing.name] - ) - case parsing.Pseudo(): - self.write_metadata_for_pseudo( - self.pseudo_instrs[thing.name] - ) - case _: - assert_never(thing) - - with self.metadata_item( - "const struct opcode_macro_expansion " - "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]", - "=", - ";", - ): - # Write macro expansion for each non-pseudo instruction - for mac in self.macro_instrs.values(): - if is_super_instruction(mac): - # Special-case the heck out of super-instructions - self.write_super_expansions(mac.name) - else: - self.write_macro_expansions( - mac.name, mac.parts, mac.cache_offset - ) - - with self.metadata_item( - "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]", "=", ";" - ): - self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') - - with self.metadata_item( - f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", - "=", - ";", - ): - for name in self.opmap: - self.out.emit(f'[{name}] = "{name}",') - - with self.metadata_item( - f"const uint8_t _PyOpcode_Caches[256]", - "=", - ";", - ): - family_member_names: set[str] = set() - for family in self.families.values(): - family_member_names.update(family.members) - for mac in self.macro_instrs.values(): - if ( - mac.cache_offset > 0 - and mac.name not in family_member_names - and not mac.name.startswith("INSTRUMENTED_") - ): - self.out.emit(f"[{mac.name}] = {mac.cache_offset},") - - deoptcodes = {} - for name, op in self.opmap.items(): - if op < 256: - deoptcodes[name] = name - for name, family in self.families.items(): - for m in family.members: - deoptcodes[m] = name - # special case: - deoptcodes["BINARY_OP_INPLACE_ADD_UNICODE"] = "BINARY_OP" - - with self.metadata_item(f"const uint8_t _PyOpcode_Deopt[256]", "=", ";"): - for opt, deopt in sorted(deoptcodes.items()): - self.out.emit(f"[{opt}] = {deopt},") - - self.out.emit("") - self.out.emit("#define EXTRA_CASES \\") - valid_opcodes = set(self.opmap.values()) - with self.out.indent(): - for op in range(256): - if op not in valid_opcodes: - self.out.emit(f"case {op}: \\") - self.out.emit(" ;\n") - - with open(pymetadata_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 0, comment="#") - - self.write_provenance_header() - - # emit specializations - specialized_ops = set() - - self.out.emit("") - self.out.emit("_specializations = {") - for name, family in self.families.items(): - with self.out.indent(): - self.out.emit(f'"{family.name}": [') - with self.out.indent(): - for m in family.members: - self.out.emit(f'"{m}",') - specialized_ops.update(family.members) - self.out.emit(f"],") - self.out.emit("}") - - # Handle special case - self.out.emit("") - self.out.emit("# An irregular case:") - self.out.emit( - '_specializations["BINARY_OP"].append(' - '"BINARY_OP_INPLACE_ADD_UNICODE")' - ) - specialized_ops.add("BINARY_OP_INPLACE_ADD_UNICODE") - - ops = sorted((id, name) for (name, id) in self.opmap.items()) - # emit specialized opmap - self.out.emit("") - with self.out.block("_specialized_opmap ="): - for op, name in ops: - if name in specialized_ops: - self.out.emit(f"'{name}': {op},") - - # emit opmap - self.out.emit("") - with self.out.block("opmap ="): - for op, name in ops: - if name not in specialized_ops: - self.out.emit(f"'{name}': {op},") - - for name in ["MIN_INSTRUMENTED_OPCODE", "HAVE_ARGUMENT"]: - self.out.emit(f"{name} = {self.markers[name]}") - - def write_pseudo_instrs(self) -> None: - """Write the IS_PSEUDO_INSTR macro""" - self.out.emit("\n\n#define IS_PSEUDO_INSTR(OP) ( \\") - for op in self.pseudos: - self.out.emit(f" ((OP) == {op}) || \\") - self.out.emit(f" 0)") - - def write_uop_items(self, make_text: typing.Callable[[str, int], str]) -> None: - """Write '#define XXX NNN' for each uop""" - counter = 300 # TODO: Avoid collision with pseudo instructions - seen = set() - - def add(name: str) -> None: - if name in seen: - return - nonlocal counter - self.out.emit(make_text(name, counter)) - counter += 1 - seen.add(name) - - # These two are first by convention - add("_EXIT_TRACE") - add("_SET_IP") - - for instr in self.instrs.values(): - # Skip ops that are also macros -- those are desugared inst()s - if instr.name not in self.macros: - add(instr.name) - - def write_macro_expansions( - self, name: str, parts: MacroParts, cache_offset: int - ) -> None: - """Write the macro expansions for a macro-instruction.""" - # TODO: Refactor to share code with write_cody(), is_viaible_uop(), etc. - offset = 0 # Cache effect offset - expansions: list[tuple[str, int, int]] = [] # [(name, size, offset), ...] - for part in parts: - if isinstance(part, Component): - # Skip specializations - if "specializing" in part.instr.annotations: - continue - # All other component instructions must be viable uops - if not part.instr.is_viable_uop() and "replaced" not in part.instr.annotations: - # This note just reminds us about macros that cannot - # be expanded to Tier 2 uops. It is not an error. - # Suppress it using 'replaced op(...)' for macros having - # manual translation in translate_bytecode_to_trace() - # in Python/optimizer.c. - if len(parts) > 1 or part.instr.name != name: - self.note( - f"Part {part.instr.name} of {name} is not a viable uop", - part.instr.inst, - ) - return - if not part.active_caches: - if part.instr.name == "_SAVE_RETURN_OFFSET": - size, offset = OPARG_SIZES["OPARG_SAVE_RETURN_OFFSET"], cache_offset - else: - size, offset = OPARG_SIZES["OPARG_FULL"], 0 - else: - # If this assert triggers, is_viable_uops() lied - assert len(part.active_caches) == 1, (name, part.instr.name) - cache = part.active_caches[0] - size, offset = cache.effect.size, cache.offset - expansions.append((part.instr.name, size, offset)) - assert len(expansions) > 0, f"Macro {name} has empty expansion?!" - self.write_expansions(name, expansions) - - def write_super_expansions(self, name: str) -> None: - """Write special macro expansions for super-instructions. - - If you get an assertion failure here, you probably have accidentally - violated one of the assumptions here. - - - A super-instruction's name is of the form FIRST_SECOND where - FIRST and SECOND are regular instructions whose name has the - form FOO_BAR. Thus, there must be exactly 3 underscores. - Example: LOAD_CONST_STORE_FAST. - - - A super-instruction's body uses `oparg1 and `oparg2`, and no - other instruction's body uses those variable names. - - - A super-instruction has no active (used) cache entries. - - In the expansion, the first instruction's operand is all but the - bottom 4 bits of the super-instruction's oparg, and the second - instruction's operand is the bottom 4 bits. We use the special - size codes OPARG_TOP and OPARG_BOTTOM for these. - """ - pieces = name.split("_") - assert len(pieces) == 4, f"{name} doesn't look like a super-instr" - name1 = "_".join(pieces[:2]) - name2 = "_".join(pieces[2:]) - assert name1 in self.instrs, f"{name1} doesn't match any instr" - assert name2 in self.instrs, f"{name2} doesn't match any instr" - instr1 = self.instrs[name1] - instr2 = self.instrs[name2] - assert not instr1.active_caches, f"{name1} has active caches" - assert not instr2.active_caches, f"{name2} has active caches" - expansions: list[tuple[str, int, int]] = [ - (name1, OPARG_SIZES["OPARG_TOP"], 0), - (name2, OPARG_SIZES["OPARG_BOTTOM"], 0), - ] - self.write_expansions(name, expansions) - - def write_expansions( - self, name: str, expansions: list[tuple[str, int, int]] - ) -> None: - pieces = [ - f"{{ {name}, {size}, {offset} }}" for name, size, offset in expansions - ] - self.out.emit( - f"[{name}] = " - f"{{ .nuops = {len(pieces)}, .uops = {{ {', '.join(pieces)} }} }}," - ) - - def emit_metadata_entry(self, name: str, fmt: str | None, flags: InstructionFlags) -> None: - flag_names = flags.names(value=True) - if not flag_names: - flag_names.append("0") - fmt_macro = "0" if fmt is None else INSTR_FMT_PREFIX + fmt - self.out.emit( - f"[{name}] = {{ true, {fmt_macro}," - f" {' | '.join(flag_names)} }}," - ) - - def write_metadata_for_inst(self, instr: Instruction) -> None: - """Write metadata for a single instruction.""" - self.emit_metadata_entry(instr.name, instr.instr_fmt, instr.instr_flags) - - def write_metadata_for_macro(self, mac: MacroInstruction) -> None: - """Write metadata for a macro-instruction.""" - self.emit_metadata_entry(mac.name, mac.instr_fmt, mac.instr_flags) - - def write_metadata_for_pseudo(self, ps: PseudoInstruction) -> None: - """Write metadata for a macro-instruction.""" - self.emit_metadata_entry(ps.name, None, ps.instr_flags) - - def write_instructions( - self, output_filename: str, emit_line_directives: bool - ) -> None: - """Write instructions to output file.""" - with open(output_filename, "w") as f: - # Create formatter - self.out = Formatter(f, 8, emit_line_directives) - - self.write_provenance_header() - - self.out.write_raw("\n") - self.out.write_raw("#ifdef TIER_TWO\n") - self.out.write_raw(" #error \"This file is for Tier 1 only\"\n") - self.out.write_raw("#endif\n") - self.out.write_raw("#define TIER_ONE 1\n") - - # Write and count instructions of all kinds - n_macros = 0 - cases = [] - for thing in self.everything: - match thing: - case parsing.InstDef(): - pass - case parsing.Macro(): - n_macros += 1 - mac = self.macro_instrs[thing.name] - cases.append((mac.name, mac)) - case parsing.Pseudo(): - pass - case _: - assert_never(thing) - cases.sort() - for _, mac in cases: - stacking.write_macro_instr(mac, self.out) - - self.out.write_raw("\n") - self.out.write_raw("#undef TIER_ONE\n") - - print( - f"Wrote {n_macros} cases to {output_filename}", - file=sys.stderr, - ) - - def write_executor_instructions( - self, executor_filename: str, emit_line_directives: bool - ) -> None: - """Generate cases for the Tier 2 interpreter.""" - n_uops = 0 - with open(executor_filename, "w") as f: - self.out = Formatter(f, 8, emit_line_directives) - self.write_provenance_header() - - self.out.write_raw("\n") - self.out.write_raw("#ifdef TIER_ONE\n") - self.out.write_raw(" #error \"This file is for Tier 2 only\"\n") - self.out.write_raw("#endif\n") - self.out.write_raw("#define TIER_TWO 2\n") - - for instr in self.instrs.values(): - if instr.is_viable_uop(): - n_uops += 1 - self.out.emit("") - with self.out.block(f"case {instr.name}:"): - if instr.instr_flags.HAS_ARG_FLAG: - self.out.emit("oparg = CURRENT_OPARG();") - stacking.write_single_instr(instr, self.out, tier=TIER_TWO) - if instr.check_eval_breaker: - self.out.emit("CHECK_EVAL_BREAKER();") - self.out.emit("break;") - - self.out.write_raw("\n") - self.out.write_raw("#undef TIER_TWO\n") - - print( - f"Wrote {n_uops} cases to {executor_filename}", - file=sys.stderr, - ) - - def write_abstract_interpreter_instructions( - self, abstract_interpreter_filename: str, emit_line_directives: bool - ) -> None: - """Generate cases for the Tier 2 abstract interpreter/analzyer.""" - with open(abstract_interpreter_filename, "w") as f: - self.out = Formatter(f, 8, emit_line_directives) - self.write_provenance_header() - for instr in self.instrs.values(): - instr = AbstractInstruction(instr.inst) - if ( - instr.is_viable_uop() - and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR - ): - self.out.emit("") - with self.out.block(f"case {instr.name}:"): - instr.write(self.out, tier=TIER_TWO) - self.out.emit("break;") - print( - f"Wrote some stuff to {abstract_interpreter_filename}", - file=sys.stderr, - ) - - -def is_super_instruction(mac: MacroInstruction) -> bool: - if ( - len(mac.parts) == 1 - and isinstance(mac.parts[0], Component) - and variable_used(mac.parts[0].instr.inst, "oparg1") - ): - assert variable_used(mac.parts[0].instr.inst, "oparg2") - return True - else: - return False - - -def main() -> None: - """Parse command line, parse input, analyze, write output.""" - args = arg_parser.parse_args() # Prints message and sys.exit(2) on error - if len(args.input) == 0: - args.input.append(DEFAULT_INPUT) - - # Raises OSError if input unreadable - a = Generator(args.input) - - a.parse() # Raises SyntaxError on failure - a.analyze() # Prints messages and sets a.errors on failure - if a.errors: - sys.exit(f"Found {a.errors} errors") - if args.viable: - # Load execution counts from bmraw.json, if it exists - a.report_non_viable_uops("bmraw.json") - return - - # These raise OSError if output can't be written - a.write_instructions(args.output, args.emit_line_directives) - - a.assign_opcode_ids() - a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h) - a.write_metadata(args.metadata, args.pymetadata) - a.write_executor_instructions(args.executor_cases, args.emit_line_directives) - a.write_abstract_interpreter_instructions( - args.abstract_interpreter_cases, args.emit_line_directives - ) - - -if __name__ == "__main__": - main() diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py new file mode 100644 index 00000000000000..5a42a05c5c2ef2 --- /dev/null +++ b/Tools/cases_generator/generators_common.py @@ -0,0 +1,215 @@ +from pathlib import Path +from typing import TextIO + +from analyzer import ( + Instruction, + Uop, + analyze_files, + Properties, + Skip, +) +from cwriter import CWriter +from typing import Callable, Mapping, TextIO, Iterator +from lexer import Token +from stack import StackOffset, Stack + + +ROOT = Path(__file__).parent.parent.parent +DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute().as_posix() + + +def root_relative_path(filename: str) -> str: + try: + return Path(filename).absolute().relative_to(ROOT).as_posix() + except ValueError: + # Not relative to root, just return original path. + return filename + + +def write_header(generator: str, sources: list[str], outfile: TextIO, comment: str = "//") -> None: + outfile.write( + f"""{comment} This file is generated by {root_relative_path(generator)} +{comment} from: +{comment} {", ".join(root_relative_path(src) for src in sources)} +{comment} Do not edit! +""" + ) + + +def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: + parens = 0 + for tkn in tkn_iter: + if tkn.kind == end and parens == 0: + return + if tkn.kind == "LPAREN": + parens += 1 + if tkn.kind == "RPAREN": + parens -= 1 + out.emit(tkn) + + +def replace_deopt( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("DEOPT_IF", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(", ") + assert inst is not None + assert inst.family is not None + out.emit(inst.family.name) + out.emit(");\n") + + +def replace_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit(") ") + c_offset = stack.peek_offset.to_c() + try: + offset = -int(c_offset) + close = ";\n" + except ValueError: + offset = None + out.emit(f"{{ stack_pointer += {c_offset}; ") + close = "; }\n" + out.emit("goto ") + if offset: + out.emit(f"pop_{offset}_") + out.emit(label) + out.emit(close) + + +def replace_decrefs( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + for var in uop.stack.inputs: + if var.name == "unused" or var.name == "null" or var.peek: + continue + if var.size != "1": + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"Py_DECREF({var.name}[_i]);\n") + out.emit("}\n") + elif var.condition: + out.emit(f"Py_XDECREF({var.name});\n") + else: + out.emit(f"Py_DECREF({var.name});\n") + + +def replace_store_sp( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + out.emit_at("", tkn) + stack.flush(out) + out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + + +def replace_check_eval_breaker( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + if not uop.properties.ends_with_eval_breaker: + out.emit_at("CHECK_EVAL_BREAKER();", tkn) + + +REPLACEMENT_FUNCTIONS = { + "DEOPT_IF": replace_deopt, + "ERROR_IF": replace_error, + "DECREF_INPUTS": replace_decrefs, + "CHECK_EVAL_BREAKER": replace_check_eval_breaker, + "STORE_SP": replace_store_sp, +} + +ReplacementFunctionType = Callable[ + [CWriter, Token, Iterator[Token], Uop, Stack, Instruction | None], None +] + + +def emit_tokens( + out: CWriter, + uop: Uop, + stack: Stack, + inst: Instruction | None, + replacement_functions: Mapping[ + str, ReplacementFunctionType + ] = REPLACEMENT_FUNCTIONS, +) -> None: + tkns = uop.body[1:-1] + if not tkns: + return + tkn_iter = iter(tkns) + out.start_line() + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text in replacement_functions: + replacement_functions[tkn.text](out, tkn, tkn_iter, uop, stack, inst) + else: + out.emit(tkn) + + +def cflags(p: Properties) -> str: + flags: list[str] = [] + if p.oparg: + flags.append("HAS_ARG_FLAG") + if p.uses_co_consts: + flags.append("HAS_CONST_FLAG") + if p.uses_co_names: + flags.append("HAS_NAME_FLAG") + if p.jumps: + flags.append("HAS_JUMP_FLAG") + if p.has_free: + flags.append("HAS_FREE_FLAG") + if p.uses_locals: + flags.append("HAS_LOCAL_FLAG") + if p.eval_breaker: + flags.append("HAS_EVAL_BREAK_FLAG") + if p.deopts: + flags.append("HAS_DEOPT_FLAG") + if not p.infallible: + flags.append("HAS_ERROR_FLAG") + if p.escapes: + flags.append("HAS_ESCAPES_FLAG") + if flags: + return " | ".join(flags) + else: + return "0" diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py deleted file mode 100644 index 149a08810e4ae5..00000000000000 --- a/Tools/cases_generator/instructions.py +++ /dev/null @@ -1,355 +0,0 @@ -import dataclasses -import re -import typing - -from flags import InstructionFlags, variable_used, variable_used_unspecialized -from formatting import ( - Formatter, - UNUSED, - list_effect_size, -) -import lexer as lx -import parsing -from parsing import StackEffect -import stacking - -BITS_PER_CODE_UNIT = 16 - - -@dataclasses.dataclass -class ActiveCacheEffect: - """Wraps a CacheEffect that is actually used, in context.""" - - effect: parsing.CacheEffect - offset: int - - -FORBIDDEN_NAMES_IN_UOPS = ( - "next_instr", - "oparg1", # Proxy for super-instructions like LOAD_FAST_LOAD_FAST - "JUMPBY", - "DISPATCH", - "TIER_ONE_ONLY", -) - - -# Interpreter tiers -TIER_ONE: typing.Final = 1 # Specializing adaptive interpreter (PEP 659) -TIER_TWO: typing.Final = 2 # Experimental tracing interpreter -Tiers: typing.TypeAlias = typing.Literal[1, 2] - - -@dataclasses.dataclass -class Instruction: - """An instruction with additional data and code.""" - - # Parts of the underlying instruction definition - inst: parsing.InstDef - name: str - annotations: list[str] - block: parsing.Block - block_text: list[str] # Block.text, less curlies, less PREDICT() calls - block_line: int # First line of block in original code - - # Computed by constructor - always_exits: str # If the block always exits, its last line; else "" - has_deopt: bool - needs_this_instr: bool - cache_offset: int - cache_effects: list[parsing.CacheEffect] - input_effects: list[StackEffect] - output_effects: list[StackEffect] - unmoved_names: frozenset[str] - instr_fmt: str - instr_flags: InstructionFlags - active_caches: list[ActiveCacheEffect] - - # Set later - family: parsing.Family | None = None - predicted: bool = False - - def __init__(self, inst: parsing.InstDef): - self.inst = inst - self.name = inst.name - self.annotations = inst.annotations - self.block = inst.block - self.block_text, self.check_eval_breaker, self.block_line = extract_block_text( - self.block - ) - self.always_exits = always_exits(self.block_text) - self.has_deopt = variable_used(self.inst, "DEOPT_IF") - self.cache_effects = [ - effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) - ] - self.cache_offset = sum(c.size for c in self.cache_effects) - self.needs_this_instr = variable_used(self.inst, "this_instr") or any(c.name != UNUSED for c in self.cache_effects) - self.input_effects = [ - effect for effect in inst.inputs if isinstance(effect, StackEffect) - ] - self.output_effects = inst.outputs # For consistency/completeness - unmoved_names: set[str] = set() - for ieffect, oeffect in zip(self.input_effects, self.output_effects): - if ieffect == oeffect and ieffect.name == oeffect.name: - unmoved_names.add(ieffect.name) - else: - break - self.unmoved_names = frozenset(unmoved_names) - - self.instr_flags = InstructionFlags.fromInstruction(inst) - - self.active_caches = [] - offset = 0 - for effect in self.cache_effects: - if effect.name != UNUSED: - self.active_caches.append(ActiveCacheEffect(effect, offset)) - offset += effect.size - - if self.instr_flags.HAS_ARG_FLAG: - fmt = "IB" - else: - fmt = "IX" - if offset: - fmt += "C" + "0" * (offset - 1) - self.instr_fmt = fmt - - def is_viable_uop(self) -> bool: - """Whether this instruction is viable as a uop.""" - dprint: typing.Callable[..., None] = lambda *args, **kwargs: None - if "FRAME" in self.name: - dprint = print - - if self.name == "_EXIT_TRACE": - return True # This has 'return frame' but it's okay - if self.name == "_SAVE_RETURN_OFFSET": - return True # Adjusts next_instr, but only in tier 1 code - if self.always_exits: - dprint(f"Skipping {self.name} because it always exits: {self.always_exits}") - return False - if len(self.active_caches) > 1: - # print(f"Skipping {self.name} because it has >1 cache entries") - return False - res = True - for forbidden in FORBIDDEN_NAMES_IN_UOPS: - # NOTE: To disallow unspecialized uops, use - # if variable_used(self.inst, forbidden): - if variable_used_unspecialized(self.inst, forbidden): - dprint(f"Skipping {self.name} because it uses {forbidden}") - res = False - return res - - def write_body( - self, - out: Formatter, - dedent: int, - active_caches: list[ActiveCacheEffect], - tier: Tiers, - family: parsing.Family | None, - ) -> None: - """Write the instruction body.""" - # Write cache effect variable declarations and initializations - for active in active_caches: - ceffect = active.effect - bits = ceffect.size * BITS_PER_CODE_UNIT - if bits == 64: - # NOTE: We assume that 64-bit data in the cache - # is always an object pointer. - # If this becomes false, we need a way to specify - # syntactically what type the cache data is. - typ = "PyObject *" - func = "read_obj" - else: - typ = f"uint{bits}_t " - func = f"read_u{bits}" - if tier == TIER_ONE: - out.emit( - f"{typ}{ceffect.name} = " - f"{func}(&this_instr[{active.offset + 1}].cache);" - ) - else: - out.emit(f"{typ}{ceffect.name} = ({typ.strip()})CURRENT_OPERAND();") - - # Write the body, substituting a goto for ERROR_IF() and other stuff - assert dedent <= 0 - extra = " " * -dedent - names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"}) - offset = 0 - context = self.block.context - assert context is not None and context.owner is not None - filename = context.owner.filename - for line in self.block_text: - out.set_lineno(self.block_line + offset, filename) - offset += 1 - if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line): - space, cond, label = m.groups() - space = extra + space - # ERROR_IF() must pop the inputs from the stack. - # The code block is responsible for DECREF()ing them. - # NOTE: If the label doesn't exist, just add it to ceval.c. - - # Don't pop common input/output effects at the bottom! - # These aren't DECREF'ed so they can stay. - ieffs = list(self.input_effects) - oeffs = list(self.output_effects) - while ( - ieffs - and oeffs - and ieffs[0] == oeffs[0] - and ieffs[0].name == oeffs[0].name - ): - ieffs.pop(0) - oeffs.pop(0) - ninputs, symbolic = list_effect_size(ieffs) - if ninputs: - label = f"pop_{ninputs}_{label}" - if tier == TIER_TWO: - label = label + "_tier_two" - if symbolic: - out.write_raw( - f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" - ) - else: - out.write_raw(f"{space}if ({cond}) goto {label};\n") - elif m := re.match(r"(\s*)DEOPT_IF\((.+)\);\s*(?://.*)?$", line): - space, cond = m.groups() - space = extra + space - target = family.name if family else self.name - out.write_raw(f"{space}DEOPT_IF({cond}, {target});\n") - elif "DEOPT" in line: - filename = context.owner.filename - lineno = context.owner.tokens[context.begin].line - print(f"{filename}:{lineno}: ERROR: DEOPT_IF() must be all on one line") - out.write_raw(extra + line) - elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): - out.reset_lineno() - space = extra + m.group(1) - for ieff in self.input_effects: - if ieff.name in names_to_skip: - continue - if ieff.size: - out.write_raw( - f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n" - ) - out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n") - out.write_raw(f"{space}}}\n") - else: - decref = "XDECREF" if ieff.cond else "DECREF" - out.write_raw(f"{space}Py_{decref}({ieff.name});\n") - else: - out.write_raw(extra + line) - out.reset_lineno() - - -InstructionOrCacheEffect = Instruction | parsing.CacheEffect - - -# Instruction used for abstract interpretation. -class AbstractInstruction(Instruction): - def __init__(self, inst: parsing.InstDef): - super().__init__(inst) - - def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: - """Write one abstract instruction, sans prologue and epilogue.""" - stacking.write_single_instr_for_abstract_interp(self, out) - - def write_body( - self, - out: Formatter, - dedent: int, - active_caches: list[ActiveCacheEffect], - tier: Tiers, - family: parsing.Family | None, - ) -> None: - pass - - -@dataclasses.dataclass -class Component: - instr: Instruction - active_caches: list[ActiveCacheEffect] - - -MacroParts = list[Component | parsing.CacheEffect] - - -@dataclasses.dataclass -class MacroInstruction: - """A macro instruction.""" - - name: str - instr_fmt: str - instr_flags: InstructionFlags - macro: parsing.Macro - parts: MacroParts - cache_offset: int - # Set later - predicted: bool = False - family: parsing.Family | None = None - - -@dataclasses.dataclass -class PseudoInstruction: - """A pseudo instruction.""" - - name: str - targets: list[Instruction | MacroInstruction] - instr_flags: InstructionFlags - - -AnyInstruction = Instruction | MacroInstruction | PseudoInstruction - - -def extract_block_text(block: parsing.Block) -> tuple[list[str], bool, int]: - # Get lines of text with proper dedent - blocklines = block.text.splitlines(True) - first_token: lx.Token = block.tokens[0] # IndexError means the context is broken - block_line = first_token.begin[0] - - # Remove blank lines from both ends - while blocklines and not blocklines[0].strip(): - blocklines.pop(0) - block_line += 1 - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - - # Remove leading and trailing braces - assert blocklines and blocklines[0].strip() == "{" - assert blocklines and blocklines[-1].strip() == "}" - blocklines.pop() - blocklines.pop(0) - block_line += 1 - - # Remove trailing blank lines - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - - # Separate CHECK_EVAL_BREAKER() macro from end - check_eval_breaker = ( - blocklines != [] and blocklines[-1].strip() == "CHECK_EVAL_BREAKER();" - ) - if check_eval_breaker: - del blocklines[-1] - - return blocklines, check_eval_breaker, block_line - - -def always_exits(lines: list[str]) -> str: - """Determine whether a block always ends in a return/goto/etc.""" - if not lines: - return "" - line = lines[-1].rstrip() - # Indent must match exactly (TODO: Do something better) - if line[:12] != " " * 12: - return "" - line = line[12:] - if line.startswith( - ( - "goto ", - "return ", - "DISPATCH", - "GO_TO_", - "Py_UNREACHABLE()", - "ERROR_IF(true, ", - ) - ): - return line - return "" diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 1185c855785939..c3c2954a42083f 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -112,7 +112,7 @@ def choice(*opts: str) -> str: char = r"\'.\'" # TODO: escape sequence CHARACTER = "CHARACTER" -comment_re = r"//.*|/\*([^*]|\*[^/])*\*/" +comment_re = r"(//.*)|/\*([^*]|\*[^/])*\*/" COMMENT = "COMMENT" newline = r"\n" @@ -234,6 +234,7 @@ def make_syntax_error( @dataclass(slots=True) class Token: + filename: str kind: str text: str begin: tuple[int, int] @@ -261,7 +262,7 @@ def width(self) -> int: def replaceText(self, txt: str) -> "Token": assert isinstance(txt, str) - return Token(self.kind, txt, self.begin, self.end) + return Token(self.filename, self.kind, txt, self.begin, self.end) def __repr__(self) -> str: b0, b1 = self.begin @@ -272,7 +273,7 @@ def __repr__(self) -> str: return f"{self.kind}({self.text!r}, {b0}:{b1}, {e0}:{e1})" -def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[Token]: +def tokenize(src: str, line: int = 1, filename: str = "") -> Iterator[Token]: linestart = -1 for m in matcher.finditer(src): start, end = m.span() @@ -323,7 +324,7 @@ def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[T else: begin = line, start - linestart if kind != "\n": - yield Token(kind, text, begin, (line, start - linestart + len(text))) + yield Token(filename, kind, text, begin, (line, start - linestart + len(text))) def to_text(tkns: list[Token], dedent: int = 0) -> str: diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index e7175e263350b2..8e5a31851c596e 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -11,3 +11,5 @@ strict = True strict_concatenate = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True +allow_redefinition = True +implicit_reexport = True diff --git a/Tools/cases_generator/opcode_id_generator.py b/Tools/cases_generator/opcode_id_generator.py new file mode 100644 index 00000000000000..dbea3d0b622c87 --- /dev/null +++ b/Tools/cases_generator/opcode_id_generator.py @@ -0,0 +1,65 @@ +"""Generate the list of opcode IDs. +Reads the instruction definitions from bytecodes.c. +Writes the IDs to opcode._ids.h by default. +""" + +import argparse +import os.path +import sys + +from analyzer import ( + Analysis, + Instruction, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, +) +from cwriter import CWriter +from typing import TextIO + + +DEFAULT_OUTPUT = ROOT / "Include/opcode_ids.h" + + +def generate_opcode_header( + filenames: list[str], analysis: Analysis, outfile: TextIO +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 0, False) + with out.header_guard("Py_OPCODE_IDS_H"): + out.emit("/* Instruction opcodes for compiled code */\n") + + def write_define(name: str, op: int) -> None: + out.emit(f"#define {name:<38} {op:>3}\n") + + for op, name in sorted([(op, name) for (name, op) in analysis.opmap.items()]): + write_define(name, op) + + out.emit("\n") + write_define("HAVE_ARGUMENT", analysis.have_arg) + write_define("MIN_INSTRUMENTED_OPCODE", analysis.min_instrumented) + + +arg_parser = argparse.ArgumentParser( + description="Generate the header file with all opcode IDs.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_opcode_header(args.input, data, outfile) diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py new file mode 100644 index 00000000000000..9b7df9a54c7b3b --- /dev/null +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -0,0 +1,386 @@ +"""Generate uop metedata. +Reads the instruction definitions from bytecodes.c. +Writes the metadata to pycore_uop_metadata.h by default. +""" + +import argparse +import os.path +import sys + +from analyzer import ( + Analysis, + Instruction, + analyze_files, + Skip, + Uop, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + cflags, + StackOffset, +) +from cwriter import CWriter +from typing import TextIO +from stack import get_stack_effect + +# Constants used instead of size for macro expansions. +# Note: 1, 2, 4 must match actual cache entry sizes. +OPARG_KINDS = { + "OPARG_FULL": 0, + "OPARG_CACHE_1": 1, + "OPARG_CACHE_2": 2, + "OPARG_CACHE_4": 4, + "OPARG_TOP": 5, + "OPARG_BOTTOM": 6, + "OPARG_SAVE_RETURN_OFFSET": 7, + # Skip 8 as the other powers of 2 are sizes + "OPARG_REPLACED": 9, +} + +FLAGS = [ + "ARG", + "CONST", + "NAME", + "JUMP", + "FREE", + "LOCAL", + "EVAL_BREAK", + "DEOPT", + "ERROR", + "ESCAPES", +] + + +def generate_flag_macros(out: CWriter) -> None: + for i, flag in enumerate(FLAGS): + out.emit(f"#define HAS_{flag}_FLAG ({1< None: + for name, value in OPARG_KINDS.items(): + out.emit(f"#define {name} {value}\n") + out.emit("\n") + + +def emit_stack_effect_function( + out: CWriter, direction: str, data: list[tuple[str, str]] +) -> None: + out.emit(f"extern int _PyOpcode_num_{direction}(int opcode, int oparg);\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit(f"int _PyOpcode_num_{direction}(int opcode, int oparg) {{\n") + out.emit("switch(opcode) {\n") + for name, effect in data: + out.emit(f"case {name}:\n") + out.emit(f" return {effect};\n") + out.emit("default:\n") + out.emit(" return -1;\n") + out.emit("}\n") + out.emit("}\n\n") + out.emit("#endif\n\n") + + +def generate_stack_effect_functions(analysis: Analysis, out: CWriter) -> None: + popped_data: list[tuple[str, str]] = [] + pushed_data: list[tuple[str, str]] = [] + for inst in analysis.instructions.values(): + stack = get_stack_effect(inst) + popped = (-stack.base_offset).to_c() + pushed = (stack.top_offset - stack.base_offset).to_c() + popped_data.append((inst.name, popped)) + pushed_data.append((inst.name, pushed)) + emit_stack_effect_function(out, "popped", sorted(popped_data)) + emit_stack_effect_function(out, "pushed", sorted(pushed_data)) + + +def generate_is_pseudo(analysis: Analysis, out: CWriter) -> None: + """Write the IS_PSEUDO_INSTR macro""" + out.emit("\n\n#define IS_PSEUDO_INSTR(OP) ( \\\n") + for op in analysis.pseudos: + out.emit(f"((OP) == {op}) || \\\n") + out.emit("0") + out.emit(")\n\n") + + +def get_format(inst: Instruction) -> str: + if inst.properties.oparg: + format = "INSTR_FMT_IB" + else: + format = "INSTR_FMT_IX" + if inst.size > 1: + format += "C" + format += "0" * (inst.size - 2) + return format + + +def generate_instruction_formats(analysis: Analysis, out: CWriter) -> None: + # Compute the set of all instruction formats. + formats: set[str] = set() + for inst in analysis.instructions.values(): + formats.add(get_format(inst)) + # Generate an enum for it + out.emit("enum InstructionFormat {\n") + next_id = 1 + for format in sorted(formats): + out.emit(f"{format} = {next_id},\n") + next_id += 1 + out.emit("};\n\n") + + +def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: + out.emit("extern const uint8_t _PyOpcode_Deopt[256];\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const uint8_t _PyOpcode_Deopt[256] = {\n") + deopts: list[tuple[str, str]] = [] + for inst in analysis.instructions.values(): + deopt = inst.name + if inst.family is not None: + deopt = inst.family.name + deopts.append((inst.name, deopt)) + deopts.append(("INSTRUMENTED_LINE", "INSTRUMENTED_LINE")) + for name, deopt in sorted(deopts): + out.emit(f"[{name}] = {deopt},\n") + out.emit("};\n\n") + out.emit("#endif // NEED_OPCODE_METADATA\n\n") + + +def generate_cache_table(analysis: Analysis, out: CWriter) -> None: + out.emit("extern const uint8_t _PyOpcode_Caches[256];\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const uint8_t _PyOpcode_Caches[256] = {\n") + for inst in analysis.instructions.values(): + if inst.family and inst.family.name != inst.name: + continue + if inst.name.startswith("INSTRUMENTED"): + continue + if inst.size > 1: + out.emit(f"[{inst.name}] = {inst.size-1},\n") + out.emit("};\n") + out.emit("#endif\n\n") + + +def generate_name_table(analysis: Analysis, out: CWriter) -> None: + table_size = 256 + len(analysis.pseudos) + out.emit(f"extern const char *_PyOpcode_OpName[{table_size}];\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit(f"const char *_PyOpcode_OpName[{table_size}] = {{\n") + names = list(analysis.instructions) + list(analysis.pseudos) + names.append("INSTRUMENTED_LINE") + for name in sorted(names): + out.emit(f'[{name}] = "{name}",\n') + out.emit("};\n") + out.emit("#endif\n\n") + + +def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: + table_size = 256 + len(analysis.pseudos) + out.emit("struct opcode_metadata {\n") + out.emit("uint8_t valid_entry;\n") + out.emit("int8_t instr_format;\n") + out.emit("int16_t flags;\n") + out.emit("};\n\n") + out.emit( + f"extern const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}];\n" + ) + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit( + f"const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}] = {{\n" + ) + for inst in sorted(analysis.instructions.values(), key=lambda t: t.name): + out.emit( + f"[{inst.name}] = {{ true, {get_format(inst)}, {cflags(inst.properties)} }},\n" + ) + for pseudo in sorted(analysis.pseudos.values(), key=lambda t: t.name): + flags = cflags(pseudo.properties) + for flag in pseudo.flags: + if flags == "0": + flags = f"{flag}_FLAG" + else: + flags += f" | {flag}_FLAG" + out.emit(f"[{pseudo.name}] = {{ true, -1, {flags} }},\n") + out.emit("};\n") + out.emit("#endif\n\n") + + +def generate_expansion_table(analysis: Analysis, out: CWriter) -> None: + expansions_table: dict[str, list[tuple[str, int, int]]] = {} + for inst in sorted(analysis.instructions.values(), key=lambda t: t.name): + offset: int = 0 # Cache effect offset + expansions: list[tuple[str, int, int]] = [] # [(name, size, offset), ...] + if inst.is_super(): + pieces = inst.name.split("_") + assert len(pieces) == 4, f"{inst.name} doesn't look like a super-instr" + name1 = "_".join(pieces[:2]) + name2 = "_".join(pieces[2:]) + assert name1 in analysis.instructions, f"{name1} doesn't match any instr" + assert name2 in analysis.instructions, f"{name2} doesn't match any instr" + instr1 = analysis.instructions[name1] + instr2 = analysis.instructions[name2] + assert ( + len(instr1.parts) == 1 + ), f"{name1} is not a good superinstruction part" + assert ( + len(instr2.parts) == 1 + ), f"{name2} is not a good superinstruction part" + expansions.append((instr1.parts[0].name, OPARG_KINDS["OPARG_TOP"], 0)) + expansions.append((instr2.parts[0].name, OPARG_KINDS["OPARG_BOTTOM"], 0)) + elif not is_viable_expansion(inst): + continue + else: + for part in inst.parts: + size = part.size + if part.name == "_SAVE_RETURN_OFFSET": + size = OPARG_KINDS["OPARG_SAVE_RETURN_OFFSET"] + if isinstance(part, Uop): + # Skip specializations + if "specializing" in part.annotations: + continue + if "replaced" in part.annotations: + size = OPARG_KINDS["OPARG_REPLACED"] + expansions.append((part.name, size, offset if size else 0)) + offset += part.size + expansions_table[inst.name] = expansions + max_uops = max(len(ex) for ex in expansions_table.values()) + out.emit(f"#define MAX_UOP_PER_EXPANSION {max_uops}\n") + out.emit("struct opcode_macro_expansion {\n") + out.emit("int nuops;\n") + out.emit( + "struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION];\n" + ) + out.emit("};\n") + out.emit( + "extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256];\n\n" + ) + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const struct opcode_macro_expansion\n") + out.emit("_PyOpcode_macro_expansion[256] = {\n") + for inst_name, expansions in expansions_table.items(): + uops = [ + f"{{ {name}, {size}, {offset} }}" for (name, size, offset) in expansions + ] + out.emit( + f'[{inst_name}] = {{ .nuops = {len(expansions)}, .uops = {{ {", ".join(uops)} }} }},\n' + ) + out.emit("};\n") + out.emit("#endif // NEED_OPCODE_METADATA\n\n") + + +def is_viable_expansion(inst: Instruction) -> bool: + "An instruction can be expanded if all its parts are viable for tier 2" + for part in inst.parts: + if isinstance(part, Uop): + # Skip specializing and replaced uops + if "specializing" in part.annotations: + continue + if "replaced" in part.annotations: + continue + if part.properties.tier_one_only or not part.is_viable(): + return False + return True + + +def generate_extra_cases(analysis: Analysis, out: CWriter) -> None: + out.emit("#define EXTRA_CASES \\\n") + valid_opcodes = set(analysis.opmap.values()) + for op in range(256): + if op not in valid_opcodes: + out.emit(f" case {op}: \\\n") + out.emit(" ;\n") + + +def generate_pseudo_targets(analysis: Analysis, out: CWriter) -> None: + table_size = len(analysis.pseudos) + max_targets = max(len(pseudo.targets) for pseudo in analysis.pseudos.values()) + out.emit("struct pseudo_targets {\n") + out.emit(f"uint8_t targets[{max_targets + 1}];\n") + out.emit("};\n") + out.emit( + f"extern const struct pseudo_targets _PyOpcode_PseudoTargets[{table_size}];\n" + ) + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit( + f"const struct pseudo_targets _PyOpcode_PseudoTargets[{table_size}] = {{\n" + ) + for pseudo in analysis.pseudos.values(): + targets = ["0"] * (max_targets + 1) + for i, target in enumerate(pseudo.targets): + targets[i] = target.name + out.emit(f"[{pseudo.name}-256] = {{ {{ {', '.join(targets)} }} }},\n") + out.emit("};\n\n") + out.emit("#endif // NEED_OPCODE_METADATA\n") + out.emit("static inline bool\n") + out.emit("is_pseudo_target(int pseudo, int target) {\n") + out.emit(f"if (pseudo < 256 || pseudo >= {256+table_size}) {{\n") + out.emit(f"return false;\n") + out.emit("}\n") + out.emit( + f"for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) {{\n" + ) + out.emit( + f"if (_PyOpcode_PseudoTargets[pseudo-256].targets[i] == target) return true;\n" + ) + out.emit("}\n") + out.emit(f"return false;\n") + out.emit("}\n\n") + + +def generate_opcode_metadata( + filenames: list[str], analysis: Analysis, outfile: TextIO +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 0, False) + with out.header_guard("Py_CORE_OPCODE_METADATA_H"): + out.emit("#ifndef Py_BUILD_CORE\n") + out.emit('# error "this header requires Py_BUILD_CORE define"\n') + out.emit("#endif\n\n") + out.emit("#include // bool\n") + out.emit('#include "opcode_ids.h"\n') + generate_is_pseudo(analysis, out) + out.emit('#include "pycore_uop_ids.h"\n') + generate_stack_effect_functions(analysis, out) + generate_instruction_formats(analysis, out) + table_size = 256 + len(analysis.pseudos) + out.emit("#define IS_VALID_OPCODE(OP) \\\n") + out.emit(f" (((OP) >= 0) && ((OP) < {table_size}) && \\\n") + out.emit(" (_PyOpcode_opcode_metadata[(OP)].valid_entry))\n\n") + generate_flag_macros(out) + generate_oparg_macros(out) + generate_metadata_table(analysis, out) + generate_expansion_table(analysis, out) + generate_name_table(analysis, out) + generate_cache_table(analysis, out) + generate_deopt_table(analysis, out) + generate_extra_cases(analysis, out) + generate_pseudo_targets(analysis, out) + + +arg_parser = argparse.ArgumentParser( + description="Generate the header file with opcode metadata.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + + +DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_opcode_metadata.h" + + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_opcode_metadata(args.input, data, outfile) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py new file mode 100644 index 00000000000000..2b77d14d21143f --- /dev/null +++ b/Tools/cases_generator/parser.py @@ -0,0 +1,66 @@ +from parsing import ( + InstDef, + Macro, + Pseudo, + Family, + Parser, + Context, + CacheEffect, + StackEffect, + InputEffect, + OpName, + AstNode, +) + + +def prettify_filename(filename: str) -> str: + # Make filename more user-friendly and less platform-specific, + # it is only used for error reporting at this point. + filename = filename.replace("\\", "/") + if filename.startswith("./"): + filename = filename[2:] + if filename.endswith(".new"): + filename = filename[:-4] + return filename + + +BEGIN_MARKER = "// BEGIN BYTECODES //" +END_MARKER = "// END BYTECODES //" + + +def parse_files(filenames: list[str]) -> list[AstNode]: + result: list[AstNode] = [] + for filename in filenames: + with open(filename) as file: + src = file.read() + + psr = Parser(src, filename=prettify_filename(filename)) + + # Skip until begin marker + while tkn := psr.next(raw=True): + if tkn.text == BEGIN_MARKER: + break + else: + raise psr.make_syntax_error( + f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" + ) + start = psr.getpos() + + # Find end marker, then delete everything after it + while tkn := psr.next(raw=True): + if tkn.text == END_MARKER: + break + del psr.tokens[psr.getpos() - 1 :] + + # Parse from start + psr.setpos(start) + thing_first_token = psr.peek() + while node := psr.definition(): + assert node is not None + result.append(node) # type: ignore[arg-type] + if not psr.eof(): + psr.backup() + raise psr.make_syntax_error( + f"Extra stuff at the end of {filename}", psr.next(True) + ) + return result diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index d36bd52b022ea9..60c185dcef58e9 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -138,12 +138,14 @@ class Family(Node): @dataclass class Pseudo(Node): name: str - targets: list[str] # opcodes this can be replaced by + flags: list[str] # instr flags to set on the pseudo instruction + targets: list[str] # opcodes this can be replaced by +AstNode = InstDef | Macro | Pseudo | Family class Parser(PLexer): @contextual - def definition(self) -> InstDef | Macro | Pseudo | Family | None: + def definition(self) -> AstNode | None: if macro := self.macro_def(): return macro if family := self.family_def(): @@ -374,19 +376,39 @@ def family_def(self) -> Family | None: ) return None + def flags(self) -> list[str]: + here = self.getpos() + if self.expect(lx.LPAREN): + if tkn := self.expect(lx.IDENTIFIER): + flags = [tkn.text] + while self.expect(lx.COMMA): + if tkn := self.expect(lx.IDENTIFIER): + flags.append(tkn.text) + else: + break + if not self.expect(lx.RPAREN): + raise self.make_syntax_error("Expected comma or right paren") + return flags + self.setpos(here) + return [] + @contextual def pseudo_def(self) -> Pseudo | None: if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "pseudo": size = None if self.expect(lx.LPAREN): if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.COMMA): + flags = self.flags() + else: + flags = [] if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): if not self.expect(lx.LBRACE): raise self.make_syntax_error("Expected {") if members := self.members(): if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Pseudo(tkn.text, members) + return Pseudo(tkn.text, flags, members) return None def members(self) -> list[str] | None: diff --git a/Tools/cases_generator/py_metadata_generator.py b/Tools/cases_generator/py_metadata_generator.py new file mode 100644 index 00000000000000..43811fdacc8a9e --- /dev/null +++ b/Tools/cases_generator/py_metadata_generator.py @@ -0,0 +1,97 @@ +"""Generate uop metedata. +Reads the instruction definitions from bytecodes.c. +Writes the metadata to pycore_uop_metadata.h by default. +""" + +import argparse + +from analyzer import ( + Analysis, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + root_relative_path, + write_header, +) +from cwriter import CWriter +from typing import TextIO + + + +DEFAULT_OUTPUT = ROOT / "Lib/_opcode_metadata.py" + + +def get_specialized(analysis: Analysis) -> set[str]: + specialized: set[str] = set() + for family in analysis.families.values(): + for member in family.members: + specialized.add(member.name) + return specialized + + +def generate_specializations(analysis: Analysis, out: CWriter) -> None: + out.emit("_specializations = {\n") + for family in analysis.families.values(): + out.emit(f'"{family.name}": [\n') + for member in family.members: + out.emit(f' "{member.name}",\n') + out.emit("],\n") + out.emit("}\n\n") + + +def generate_specialized_opmap(analysis: Analysis, out: CWriter) -> None: + out.emit("_specialized_opmap = {\n") + names = [] + for family in analysis.families.values(): + for member in family.members: + if member.name == family.name: + continue + names.append(member.name) + for name in sorted(names): + out.emit(f"'{name}': {analysis.opmap[name]},\n") + out.emit("}\n\n") + + +def generate_opmap(analysis: Analysis, out: CWriter) -> None: + specialized = get_specialized(analysis) + out.emit("opmap = {\n") + for inst, op in analysis.opmap.items(): + if inst not in specialized: + out.emit(f"'{inst}': {analysis.opmap[inst]},\n") + out.emit("}\n\n") + + +def generate_py_metadata( + filenames: list[str], analysis: Analysis, outfile: TextIO +) -> None: + write_header(__file__, filenames, outfile, "#") + out = CWriter(outfile, 0, False) + generate_specializations(analysis, out) + generate_specialized_opmap(analysis, out) + generate_opmap(analysis, out) + out.emit(f"HAVE_ARGUMENT = {analysis.have_arg}\n") + out.emit(f"MIN_INSTRUMENTED_OPCODE = {analysis.min_instrumented}\n") + + +arg_parser = argparse.ArgumentParser( + description="Generate the Python file with opcode metadata.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_py_metadata(args.input, data, outfile) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py new file mode 100644 index 00000000000000..d351037a663ca2 --- /dev/null +++ b/Tools/cases_generator/stack.py @@ -0,0 +1,206 @@ +import re +from analyzer import StackItem, Instruction, Uop +from dataclasses import dataclass +from cwriter import CWriter + + +def maybe_parenthesize(sym: str) -> str: + """Add parentheses around a string if it contains an operator + and is not already parenthesized. + + An exception is made for '*' which is common and harmless + in the context where the symbolic size is used. + """ + if sym.startswith("(") and sym.endswith(")"): + return sym + if re.match(r"^[\s\w*]+$", sym): + return sym + else: + return f"({sym})" + + +def var_size(var: StackItem) -> str: + if var.condition: + # Special case simplification + if var.condition == "oparg & 1" and var.size == "1": + return f"({var.condition})" + else: + return f"(({var.condition}) ? {var.size} : 0)" + else: + return var.size + +@dataclass +class StackOffset: + "The stack offset of the virtual base of the stack from the physical stack pointer" + + popped: list[str] + pushed: list[str] + + @staticmethod + def empty() -> "StackOffset": + return StackOffset([], []) + + def pop(self, item: StackItem) -> None: + self.popped.append(var_size(item)) + + def push(self, item: StackItem) -> None: + self.pushed.append(var_size(item)) + + def __sub__(self, other: "StackOffset") -> "StackOffset": + return StackOffset( + self.popped + other.pushed, + self.pushed + other.popped + ) + + def __neg__(self) -> "StackOffset": + return StackOffset(self.pushed, self.popped) + + def simplify(self) -> None: + "Remove matching values from both the popped and pushed list" + if not self.popped or not self.pushed: + return + # Sort the list so the lexically largest element is last. + popped = sorted(self.popped) + pushed = sorted(self.pushed) + self.popped = [] + self.pushed = [] + while popped and pushed: + pop = popped.pop() + push = pushed.pop() + if pop == push: + pass + elif pop > push: + # if pop > push, there can be no element in pushed matching pop. + self.popped.append(pop) + pushed.append(push) + else: + self.pushed.append(push) + popped.append(pop) + self.popped.extend(popped) + self.pushed.extend(pushed) + + def to_c(self) -> str: + self.simplify() + int_offset = 0 + symbol_offset = "" + for item in self.popped: + try: + int_offset -= int(item) + except ValueError: + symbol_offset += f" - {maybe_parenthesize(item)}" + for item in self.pushed: + try: + int_offset += int(item) + except ValueError: + symbol_offset += f" + {maybe_parenthesize(item)}" + if symbol_offset and not int_offset: + res = symbol_offset + else: + res = f"{int_offset}{symbol_offset}" + if res.startswith(" + "): + res = res[3:] + if res.startswith(" - "): + res = "-" + res[3:] + return res + + def clear(self) -> None: + self.popped = [] + self.pushed = [] + + +class SizeMismatch(Exception): + pass + + +class Stack: + def __init__(self) -> None: + self.top_offset = StackOffset.empty() + self.base_offset = StackOffset.empty() + self.peek_offset = StackOffset.empty() + self.variables: list[StackItem] = [] + self.defined: set[str] = set() + + def pop(self, var: StackItem) -> str: + self.top_offset.pop(var) + if not var.peek: + self.peek_offset.pop(var) + indirect = "&" if var.is_array() else "" + if self.variables: + popped = self.variables.pop() + if popped.size != var.size: + raise SizeMismatch( + f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " + f"Expected {var.size} got {popped.size}" + ) + if popped.name == var.name: + return "" + elif popped.name == "unused": + self.defined.add(var.name) + return ( + f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" + ) + elif var.name == "unused": + return "" + else: + self.defined.add(var.name) + return f"{var.name} = {popped.name};\n" + self.base_offset.pop(var) + if var.name == "unused": + return "" + else: + self.defined.add(var.name) + cast = f"({var.type})" if (not indirect and var.type) else "" + assign = ( + f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];" + ) + if var.condition: + return f"if ({var.condition}) {{ {assign} }}\n" + return f"{assign}\n" + + def push(self, var: StackItem) -> str: + self.variables.append(var) + if var.is_array() and var.name not in self.defined and var.name != "unused": + c_offset = self.top_offset.to_c() + self.top_offset.push(var) + self.defined.add(var.name) + return f"{var.name} = &stack_pointer[{c_offset}];\n" + else: + self.top_offset.push(var) + return "" + + def flush(self, out: CWriter) -> None: + for var in self.variables: + if not var.peek: + cast = "(PyObject *)" if var.type else "" + if var.name != "unused" and not var.is_array(): + if var.condition: + out.emit(f"if ({var.condition}) ") + out.emit( + f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n" + ) + self.base_offset.push(var) + if self.base_offset.to_c() != self.top_offset.to_c(): + print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) + assert False + number = self.base_offset.to_c() + if number != "0": + out.emit(f"stack_pointer += {number};\n") + self.variables = [] + self.base_offset.clear() + self.top_offset.clear() + self.peek_offset.clear() + + def as_comment(self) -> str: + return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" + + +def get_stack_effect(inst: Instruction) -> Stack: + stack = Stack() + for uop in inst.parts: + if not isinstance(uop, Uop): + continue + for var in reversed(uop.stack.inputs): + stack.pop(var) + for i, var in enumerate(uop.stack.outputs): + stack.push(var) + return stack diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py deleted file mode 100644 index 123e38c524f49d..00000000000000 --- a/Tools/cases_generator/stacking.py +++ /dev/null @@ -1,534 +0,0 @@ -import dataclasses -import typing - -from flags import variable_used_unspecialized -from formatting import ( - Formatter, - UNUSED, - maybe_parenthesize, - parenthesize_cond, -) -from instructions import ( - ActiveCacheEffect, - Instruction, - MacroInstruction, - Component, - Tiers, - TIER_ONE, -) -from parsing import StackEffect, CacheEffect, Family - - -@dataclasses.dataclass -class StackOffset: - """Represent the stack offset for a PEEK or POKE. - - - At stack_pointer[0], deep and high are both empty. - (Note that that is an invalid stack reference.) - - Below stack top, only deep is non-empty. - - Above stack top, only high is non-empty. - - In complex cases, both deep and high may be non-empty. - - All this would be much simpler if all stack entries were the same - size, but with conditional and array effects, they aren't. - The offsets are each represented by a list of StackEffect objects. - The name in the StackEffects is unused. - """ - - deep: list[StackEffect] = dataclasses.field(default_factory=list) - high: list[StackEffect] = dataclasses.field(default_factory=list) - - def clone(self) -> "StackOffset": - return StackOffset(list(self.deep), list(self.high)) - - def negate(self) -> "StackOffset": - return StackOffset(list(self.high), list(self.deep)) - - def deeper(self, eff: StackEffect) -> None: - if eff in self.high: - self.high.remove(eff) - else: - self.deep.append(eff) - - def higher(self, eff: StackEffect) -> None: - if eff in self.deep: - self.deep.remove(eff) - else: - self.high.append(eff) - - def as_terms(self) -> list[tuple[str, str]]: - num = 0 - terms: list[tuple[str, str]] = [] - for eff in self.deep: - if eff.size: - terms.append(("-", maybe_parenthesize(eff.size))) - elif eff.cond and eff.cond not in ("0", "1"): - terms.append(("-", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) - elif eff.cond != "0": - num -= 1 - for eff in self.high: - if eff.size: - terms.append(("+", maybe_parenthesize(eff.size))) - elif eff.cond and eff.cond not in ("0", "1"): - terms.append(("+", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) - elif eff.cond != "0": - num += 1 - if num < 0: - terms.insert(0, ("-", str(-num))) - elif num > 0: - terms.append(("+", str(num))) - return terms - - def as_index(self) -> str: - terms = self.as_terms() - return make_index(terms) - - def equivalent_to(self, other: "StackOffset") -> bool: - if self.deep == other.deep and self.high == other.high: - return True - deep = list(self.deep) - for x in other.deep: - try: - deep.remove(x) - except ValueError: - return False - if deep: - return False - high = list(self.high) - for x in other.high: - try: - high.remove(x) - except ValueError: - return False - if high: - return False - return True - - -def make_index(terms: list[tuple[str, str]]) -> str: - # Produce an index expression from the terms honoring PEP 8, - # surrounding binary ops with spaces but not unary minus - index = "" - for sign, term in terms: - if index: - index += f" {sign} {term}" - elif sign == "+": - index = term - else: - index = sign + term - return index or "0" - - -@dataclasses.dataclass -class StackItem: - offset: StackOffset - effect: StackEffect - - def as_variable(self, lax: bool = False) -> str: - """Return e.g. stack_pointer[-1].""" - terms = self.offset.as_terms() - if self.effect.size: - terms.insert(0, ("+", "stack_pointer")) - index = make_index(terms) - if self.effect.size: - res = index - else: - res = f"stack_pointer[{index}]" - if not lax: - # Check that we're not reading or writing above stack top. - # Skip this for output variable initialization (lax=True). - assert ( - self.effect in self.offset.deep and not self.offset.high - ), f"Push or pop above current stack level: {res}" - return res - - def as_stack_effect(self, lax: bool = False) -> StackEffect: - return StackEffect( - self.as_variable(lax=lax), - self.effect.type if self.effect.size else "", - self.effect.cond, - self.effect.size, - ) - - -@dataclasses.dataclass -class CopyItem: - src: StackItem - dst: StackItem - - -class EffectManager: - """Manage stack effects and offsets for an instruction.""" - - instr: Instruction - active_caches: list[ActiveCacheEffect] - peeks: list[StackItem] - pokes: list[StackItem] - copies: list[CopyItem] # See merge() - # Track offsets from stack pointer - min_offset: StackOffset - final_offset: StackOffset - # Link to previous manager - pred: "EffectManager | None" = None - - def __init__( - self, - instr: Instruction, - active_caches: list[ActiveCacheEffect], - pred: "EffectManager | None" = None, - ): - self.instr = instr - self.active_caches = active_caches - self.peeks = [] - self.pokes = [] - self.copies = [] - self.final_offset = pred.final_offset.clone() if pred else StackOffset() - for eff in reversed(instr.input_effects): - self.final_offset.deeper(eff) - self.peeks.append(StackItem(offset=self.final_offset.clone(), effect=eff)) - self.min_offset = self.final_offset.clone() - for eff in instr.output_effects: - self.pokes.append(StackItem(offset=self.final_offset.clone(), effect=eff)) - self.final_offset.higher(eff) - - self.pred = pred - while pred: - # Replace push(x) + pop(y) with copy(x, y). - # Check that the sources and destinations are disjoint. - sources: set[str] = set() - destinations: set[str] = set() - while ( - pred.pokes - and self.peeks - and pred.pokes[-1].effect == self.peeks[0].effect - ): - src = pred.pokes.pop(-1) - dst = self.peeks.pop(0) - assert src.offset.equivalent_to(dst.offset), (src, dst) - pred.final_offset.deeper(src.effect) - if dst.effect.name != src.effect.name: - if dst.effect.name != UNUSED: - destinations.add(dst.effect.name) - if src.effect.name != UNUSED: - sources.add(src.effect.name) - self.copies.append(CopyItem(src, dst)) - # TODO: Turn this into an error (pass an Analyzer instance?) - assert sources & destinations == set(), ( - pred.instr.name, - self.instr.name, - sources, - destinations, - ) - # See if we can get more copies of a earlier predecessor. - if self.peeks and not pred.pokes and not pred.peeks: - pred = pred.pred - else: - pred = None # Break - - # Fix up patterns of copies through UNUSED, - # e.g. cp(a, UNUSED) + cp(UNUSED, b) -> cp(a, b). - if any(copy.src.effect.name == UNUSED for copy in self.copies): - pred = self.pred - while pred is not None: - for copy in self.copies: - if copy.src.effect.name == UNUSED: - for pred_copy in pred.copies: - if pred_copy.dst == copy.src: - copy.src = pred_copy.src - break - pred = pred.pred - - def adjust_deeper(self, eff: StackEffect) -> None: - for peek in self.peeks: - peek.offset.deeper(eff) - for poke in self.pokes: - poke.offset.deeper(eff) - for copy in self.copies: - copy.src.offset.deeper(eff) - copy.dst.offset.deeper(eff) - self.min_offset.deeper(eff) - self.final_offset.deeper(eff) - - def adjust_higher(self, eff: StackEffect) -> None: - for peek in self.peeks: - peek.offset.higher(eff) - for poke in self.pokes: - poke.offset.higher(eff) - for copy in self.copies: - copy.src.offset.higher(eff) - copy.dst.offset.higher(eff) - self.min_offset.higher(eff) - self.final_offset.higher(eff) - - def adjust(self, offset: StackOffset) -> None: - deep = list(offset.deep) - high = list(offset.high) - for down in deep: - self.adjust_deeper(down) - for up in high: - self.adjust_higher(up) - - def adjust_inverse(self, offset: StackOffset) -> None: - deep = list(offset.deep) - high = list(offset.high) - for down in deep: - self.adjust_higher(down) - for up in high: - self.adjust_deeper(up) - - def collect_vars(self) -> dict[str, StackEffect]: - """Collect all variables, skipping unused ones.""" - vars: dict[str, StackEffect] = {} - - def add(eff: StackEffect) -> None: - if eff.name != UNUSED: - if eff.name in vars: - # TODO: Make this an error - assert vars[eff.name] == eff, ( - self.instr.name, - eff.name, - vars[eff.name], - eff, - ) - else: - vars[eff.name] = eff - - for copy in self.copies: - add(copy.src.effect) - add(copy.dst.effect) - for peek in self.peeks: - add(peek.effect) - for poke in self.pokes: - add(poke.effect) - - return vars - - -def less_than(a: StackOffset, b: StackOffset) -> bool: - # TODO: Handle more cases - if a.high != b.high: - return False - return a.deep[: len(b.deep)] == b.deep - - -def get_managers(parts: list[Component]) -> list[EffectManager]: - managers: list[EffectManager] = [] - pred: EffectManager | None = None - for part in parts: - mgr = EffectManager(part.instr, part.active_caches, pred) - managers.append(mgr) - pred = mgr - return managers - - -def get_stack_effect_info_for_macro(mac: MacroInstruction) -> tuple[str, str]: - """Get the stack effect info for a macro instruction. - - Returns a tuple (popped, pushed) where each is a string giving a - symbolic expression for the number of values popped/pushed. - """ - parts = [part for part in mac.parts if isinstance(part, Component)] - managers = get_managers(parts) - popped = StackOffset() - for mgr in managers: - if less_than(mgr.min_offset, popped): - popped = mgr.min_offset.clone() - # Compute pushed = final - popped - pushed = managers[-1].final_offset.clone() - for effect in popped.deep: - pushed.higher(effect) - for effect in popped.high: - pushed.deeper(effect) - return popped.negate().as_index(), pushed.as_index() - - -def write_single_instr( - instr: Instruction, out: Formatter, tier: Tiers = TIER_ONE -) -> None: - try: - write_components( - [Component(instr, instr.active_caches)], - out, - tier, - 0, - instr.family, - ) - except AssertionError as err: - raise AssertionError(f"Error writing instruction {instr.name}") from err - - -def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: - parts = [ - part - for part in mac.parts - if isinstance(part, Component) and part.instr.name != "_SET_IP" - ] - out.emit("") - with out.block(f"TARGET({mac.name})"): - needs_this = any(part.instr.needs_this_instr for part in parts) - if needs_this and not mac.predicted: - out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;") - else: - out.emit(f"frame->instr_ptr = next_instr;") - out.emit(f"next_instr += {mac.cache_offset+1};") - out.emit(f"INSTRUCTION_STATS({mac.name});") - if mac.predicted: - out.emit(f"PREDICTED({mac.name});") - if needs_this: - out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {mac.cache_offset+1};") - out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) - try: - next_instr_is_set = write_components( - parts, out, TIER_ONE, mac.cache_offset, mac.family - ) - except AssertionError as err: - raise AssertionError(f"Error writing macro {mac.name}") from err - if not parts[-1].instr.always_exits: - if parts[-1].instr.check_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();") - out.emit("DISPATCH();") - - -def write_components( - parts: list[Component], - out: Formatter, - tier: Tiers, - cache_offset: int, - family: Family | None, -) -> bool: - managers = get_managers(parts) - - all_vars: dict[str, StackEffect] = {} - for mgr in managers: - for name, eff in mgr.collect_vars().items(): - if name in all_vars: - # TODO: Turn this into an error -- variable conflict - assert all_vars[name] == eff, ( - name, - mgr.instr.name, - all_vars[name], - eff, - ) - else: - all_vars[name] = eff - - # Declare all variables - for name, eff in all_vars.items(): - out.declare(eff, None) - - next_instr_is_set = False - for mgr in managers: - if len(parts) > 1: - out.emit(f"// {mgr.instr.name}") - - for copy in mgr.copies: - copy_src_effect = copy.src.effect - if copy_src_effect.name != copy.dst.effect.name: - if copy_src_effect.name == UNUSED: - copy_src_effect = copy.src.as_stack_effect() - out.assign(copy.dst.effect, copy_src_effect) - for peek in mgr.peeks: - out.assign( - peek.effect, - peek.as_stack_effect(), - ) - # Initialize array outputs - for poke in mgr.pokes: - if poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.assign( - poke.effect, - poke.as_stack_effect(lax=True), - ) - - if mgr.instr.name in ("_PUSH_FRAME", "_POP_FRAME"): - # Adjust stack to min_offset. - # This means that all input effects of this instruction - # are materialized, but not its output effects. - # That's as intended, since these two are so special. - out.stack_adjust(mgr.min_offset.deep, mgr.min_offset.high) - # However, for tier 2, pretend the stack is at final offset. - mgr.adjust_inverse(mgr.final_offset) - if tier == TIER_ONE: - # TODO: Check in analyzer that _{PUSH,POP}_FRAME is last. - assert ( - mgr is managers[-1] - ), f"Expected {mgr.instr.name!r} to be the last uop" - assert_no_pokes(managers) - - if mgr.instr.name == "_SAVE_RETURN_OFFSET": - next_instr_is_set = True - if tier == TIER_ONE: - assert_no_pokes(managers) - - if len(parts) == 1: - mgr.instr.write_body(out, 0, mgr.active_caches, tier, family) - else: - with out.block(""): - mgr.instr.write_body(out, -4, mgr.active_caches, tier, family) - - if mgr is managers[-1] and not next_instr_is_set and not mgr.instr.always_exits: - # Adjust the stack to its final depth, *then* write the - # pokes for all preceding uops. - # Note that for array output effects we may still write - # past the stack top. - out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) - write_all_pokes(mgr.final_offset, managers, out) - - return next_instr_is_set - - -def assert_no_pokes(managers: list[EffectManager]) -> None: - for mgr in managers: - for poke in mgr.pokes: - if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - assert ( - poke.effect.name == UNUSED - ), f"Unexpected poke of {poke.effect.name} in {mgr.instr.name!r}" - - -def write_all_pokes( - offset: StackOffset, managers: list[EffectManager], out: Formatter -) -> None: - # Emit all remaining pushes (pokes) - for m in managers: - m.adjust_inverse(offset) - write_pokes(m, out) - - -def write_pokes(mgr: EffectManager, out: Formatter) -> None: - for poke in mgr.pokes: - if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.assign( - poke.as_stack_effect(), - poke.effect, - ) - - -def write_single_instr_for_abstract_interp(instr: Instruction, out: Formatter) -> None: - try: - _write_components_for_abstract_interp( - [Component(instr, instr.active_caches)], - out, - ) - except AssertionError as err: - raise AssertionError( - f"Error writing abstract instruction {instr.name}" - ) from err - - -def _write_components_for_abstract_interp( - parts: list[Component], - out: Formatter, -) -> None: - managers = get_managers(parts) - for mgr in managers: - if mgr is managers[-1]: - out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) - mgr.adjust_inverse(mgr.final_offset) - # NULL out the output stack effects - for poke in mgr.pokes: - if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.emit( - f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" - f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);" - ) diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py new file mode 100644 index 00000000000000..44a699c92bbd22 --- /dev/null +++ b/Tools/cases_generator/target_generator.py @@ -0,0 +1,54 @@ +"""Generate targets for computed goto dispatch +Reads the instruction definitions from bytecodes.c. +Writes the table to opcode_targets.h by default. +""" + +import argparse + +from analyzer import ( + Analysis, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, +) +from cwriter import CWriter +from typing import TextIO + + +DEFAULT_OUTPUT = ROOT / "Python/opcode_targets.h" + + +def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: + """Write header file that defines the jump target table""" + targets = ["&&_unknown_opcode,\n"] * 256 + for name, op in analysis.opmap.items(): + if op < 256: + targets[op] = f"&&TARGET_{name},\n" + out.emit("static void *opcode_targets[256] = {\n") + for target in targets: + out.emit(target) + out.emit("};\n") + +arg_parser = argparse.ArgumentParser( + description="Generate the file with dispatch targets.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + out = CWriter(outfile, 0, False) + write_opcode_targets(data, out) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py new file mode 100644 index 00000000000000..aba36ec74e5766 --- /dev/null +++ b/Tools/cases_generator/tier1_generator.py @@ -0,0 +1,200 @@ +"""Generate the main interpreter switch. +Reads the instruction definitions from bytecodes.c. +Writes the cases to generated_cases.c.h, which is #included in ceval.c. +""" + +import argparse +import os.path +import sys + +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + StackItem, + analysis_error, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + emit_tokens, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import StackOffset, Stack, SizeMismatch + + +DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h" + + +FOOTER = "#undef TIER_ONE\n" + + +def declare_variables(inst: Instruction, out: CWriter) -> None: + variables = {"unused"} + for uop in inst.parts: + if isinstance(uop, Uop): + for var in reversed(uop.stack.inputs): + if var.name not in variables: + type = var.type if var.type else "PyObject *" + variables.add(var.name) + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + for var in uop.stack.outputs: + if var.name not in variables: + variables.add(var.name) + type = var.type if var.type else "PyObject *" + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + + +def write_uop( + uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool +) -> int: + # out.emit(stack.as_comment() + "\n") + if isinstance(uop, Skip): + entries = "entries" if uop.size > 1 else "entry" + out.emit(f"/* Skip {uop.size} cache {entries} */\n") + return offset + uop.size + try: + out.start_line() + if braces: + out.emit(f"// {uop.name}\n") + for var in reversed(uop.stack.inputs): + out.emit(stack.pop(var)) + if braces: + out.emit("{\n") + if not uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = "PyObject *" + reader = "read_obj" + else: + type = f"uint{cache.size*16}_t " + reader = f"read_u{cache.size*16}" + out.emit( + f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" + ) + offset += cache.size + emit_tokens(out, uop, stack, inst) + if uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + if braces: + out.start_line() + out.emit("}\n") + # out.emit(stack.as_comment() + "\n") + return offset + except SizeMismatch as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +def uses_this(inst: Instruction) -> bool: + if inst.properties.needs_this: + return True + for uop in inst.parts: + if isinstance(uop, Skip): + continue + for cache in uop.caches: + if cache.name != "unused": + return True + return False + + +def generate_tier1( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + outfile.write( + """ +#ifdef TIER_TWO + #error "This file is for Tier 1 only" +#endif +#define TIER_ONE 1 +""" + ) + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, inst in sorted(analysis.instructions.items()): + needs_this = uses_this(inst) + out.emit("\n") + out.emit(f"TARGET({name}) {{\n") + if needs_this and not inst.is_target: + out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") + else: + out.emit(f"frame->instr_ptr = next_instr;\n") + out.emit(f"next_instr += {inst.size};\n") + out.emit(f"INSTRUCTION_STATS({name});\n") + if inst.is_target: + out.emit(f"PREDICTED({name});\n") + if needs_this: + out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n") + if inst.family is not None: + out.emit( + f"static_assert({inst.family.size} == {inst.size-1}" + ', "incorrect cache size");\n' + ) + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for part in inst.parts: + # Only emit braces if more than one uop + insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 + offset = write_uop(part, out, offset, stack, inst, insert_braces) + out.start_line() + if not inst.parts[-1].properties.always_exits: + stack.flush(out) + if inst.parts[-1].properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();\n") + out.emit("DISPATCH();\n") + out.start_line() + out.emit("}") + out.emit("\n") + outfile.write(FOOTER) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + + +def generate_tier1_from_files( + filenames: list[str], outfilename: str, lines: bool +) -> None: + data = analyze_files(filenames) + with open(outfilename, "w") as outfile: + generate_tier1(filenames, data, outfile, lines) + + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tier1(args.input, data, outfile, args.emit_line_directives) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py new file mode 100644 index 00000000000000..7897b89b2752a7 --- /dev/null +++ b/Tools/cases_generator/tier2_generator.py @@ -0,0 +1,196 @@ +"""Generate the cases for the tier 2 interpreter. +Reads the instruction definitions from bytecodes.c. +Writes the cases to executor_cases.c.h, which is #included in ceval.c. +""" + +import argparse +import os.path +import sys + +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + StackItem, + analysis_error, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + emit_tokens, + emit_to, + REPLACEMENT_FUNCTIONS, +) +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import StackOffset, Stack, SizeMismatch + +DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h" + + +def declare_variables(uop: Uop, out: CWriter) -> None: + variables = {"unused"} + for var in reversed(uop.stack.inputs): + if var.name not in variables: + type = var.type if var.type else "PyObject *" + variables.add(var.name) + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + for var in uop.stack.outputs: + if var.name not in variables: + variables.add(var.name) + type = var.type if var.type else "PyObject *" + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + + +def tier2_replace_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit(") ") + c_offset = stack.peek_offset.to_c() + try: + offset = -int(c_offset) + close = ";\n" + except ValueError: + offset = None + out.emit(f"{{ stack_pointer += {c_offset}; ") + close = "; }\n" + out.emit("goto ") + if offset: + out.emit(f"pop_{offset}_") + out.emit(label + "_tier_two") + out.emit(close) + + +def tier2_replace_deopt( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(") goto deoptimize;\n") + + +TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() +TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error +TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt + + +def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: + try: + out.start_line() + if uop.properties.oparg: + out.emit("oparg = CURRENT_OPARG();\n") + for var in reversed(uop.stack.inputs): + out.emit(stack.pop(var)) + if not uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = cast = "PyObject *" + else: + type = f"uint{cache.size*16}_t " + cast = f"uint{cache.size*16}_t" + out.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") + emit_tokens(out, uop, stack, None, TIER2_REPLACEMENT_FUNCTIONS) + if uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + except SizeMismatch as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +SKIPS = ("_EXTENDED_ARG",) + + +def generate_tier2( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + outfile.write( + """ +#ifdef TIER_ONE + #error "This file is for Tier 2 only" +#endif +#define TIER_TWO 2 +""" + ) + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, uop in analysis.uops.items(): + if uop.properties.tier_one_only: + continue + if uop.is_super(): + continue + if not uop.is_viable(): + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + continue + out.emit(f"case {uop.name}: {{\n") + declare_variables(uop, out) + stack = Stack() + write_uop(uop, out, stack) + out.start_line() + if not uop.properties.always_exits: + stack.flush(out) + if uop.properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();\n") + out.emit("break;\n") + out.start_line() + out.emit("}") + out.emit("\n\n") + outfile.write("#undef TIER_TWO\n") + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the tier 2 interpreter.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tier2(args.input, data, outfile, args.emit_line_directives) diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py new file mode 100644 index 00000000000000..633249f1c6b1fe --- /dev/null +++ b/Tools/cases_generator/uop_id_generator.py @@ -0,0 +1,80 @@ +"""Generate the list of uop IDs. +Reads the instruction definitions from bytecodes.c. +Writes the IDs to pycore_uop_ids.h by default. +""" + +import argparse +import os.path +import sys + +from analyzer import ( + Analysis, + Instruction, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, +) +from cwriter import CWriter +from typing import TextIO + + +DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_ids.h" + + +def generate_uop_ids( + filenames: list[str], analysis: Analysis, outfile: TextIO, distinct_namespace: bool +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 0, False) + with out.header_guard("Py_CORE_UOP_IDS_H"): + next_id = 1 if distinct_namespace else 300 + # These two are first by convention + out.emit(f"#define _EXIT_TRACE {next_id}\n") + next_id += 1 + out.emit(f"#define _SET_IP {next_id}\n") + next_id += 1 + PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP"} + + for uop in analysis.uops.values(): + if uop.name in PRE_DEFINED: + continue + if uop.properties.tier_one_only: + continue + if uop.implicitly_created and not distinct_namespace: + out.emit(f"#define {uop.name} {uop.name[1:]}\n") + else: + out.emit(f"#define {uop.name} {next_id}\n") + next_id += 1 + + out.emit(f"#define MAX_UOP_ID {next_id-1}\n") + + +arg_parser = argparse.ArgumentParser( + description="Generate the header file with all uop IDs.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) +arg_parser.add_argument( + "-n", + "--namespace", + help="Give uops a distinct namespace", + action="store_true", +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_uop_ids(args.input, data, outfile, args.namespace) diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py new file mode 100644 index 00000000000000..d4f3a096d2acc1 --- /dev/null +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -0,0 +1,73 @@ +"""Generate uop metedata. +Reads the instruction definitions from bytecodes.c. +Writes the metadata to pycore_uop_metadata.h by default. +""" + +import argparse + +from analyzer import ( + Analysis, + analyze_files, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + cflags, +) +from cwriter import CWriter +from typing import TextIO + + +DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h" + + +def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: + out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") + for uop in analysis.uops.values(): + if uop.is_viable() and not uop.properties.tier_one_only: + out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") + + out.emit("};\n\n") + out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {\n") + for uop in sorted(analysis.uops.values(), key=lambda t: t.name): + if uop.is_viable() and not uop.properties.tier_one_only: + out.emit(f'[{uop.name}] = "{uop.name}",\n') + out.emit("};\n") + out.emit("#endif // NEED_OPCODE_METADATA\n\n") + + +def generate_uop_metadata( + filenames: list[str], analysis: Analysis, outfile: TextIO +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 0, False) + with out.header_guard("Py_CORE_UOP_METADATA_H"): + out.emit("#include \n") + out.emit('#include "pycore_uop_ids.h"\n') + generate_names_and_flags(analysis, out) + + +arg_parser = argparse.ArgumentParser( + description="Generate the header file with uop metadata.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_uop_metadata(args.input, data, outfile) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index c0830864175adf..f6f95580f1a177 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -13,7 +13,6 @@ import collections import contextlib import copy -import cpp import dataclasses as dc import enum import functools @@ -50,6 +49,13 @@ overload, ) + +# Local imports. +import libclinic +import libclinic.cpp +from libclinic import ClinicError + + # TODO: # # soon: @@ -61,25 +67,6 @@ # and keyword-only # -version = '1' - -NO_VARARG = "PY_SSIZE_T_MAX" -CLINIC_PREFIX = "__clinic_" -CLINIC_PREFIXED_ARGS = { - "_keywords", - "_parser", - "args", - "argsbuf", - "fastargs", - "kwargs", - "kwnames", - "nargs", - "noptargs", - "return_value", -} - -# '#include "header.h" // reason': column of '//' comment -INCLUDE_COMMENT_COLUMN = 35 # match '#define Py_LIMITED_API' LIMITED_CAPI_REGEX = re.compile(r'#define +Py_LIMITED_API') @@ -105,65 +92,8 @@ def __repr__(self) -> str: NULL = Null() -sig_end_marker = '--' - -Appender = Callable[[str], None] -Outputter = Callable[[], str] TemplateDict = dict[str, str] -class _TextAccumulator(NamedTuple): - text: list[str] - append: Appender - output: Outputter - -def _text_accumulator() -> _TextAccumulator: - text: list[str] = [] - def output() -> str: - s = ''.join(text) - text.clear() - return s - return _TextAccumulator(text, text.append, output) - - -class TextAccumulator(NamedTuple): - append: Appender - output: Outputter - -def text_accumulator() -> TextAccumulator: - """ - Creates a simple text accumulator / joiner. - - Returns a pair of callables: - append, output - "append" appends a string to the accumulator. - "output" returns the contents of the accumulator - joined together (''.join(accumulator)) and - empties the accumulator. - """ - text, append, output = _text_accumulator() - return TextAccumulator(append, output) - - -@dc.dataclass -class ClinicError(Exception): - message: str - _: dc.KW_ONLY - lineno: int | None = None - filename: str | None = None - - def __post_init__(self) -> None: - super().__init__(self.message) - - def report(self, *, warn_only: bool = False) -> str: - msg = "Warning" if warn_only else "Error" - if self.filename is not None: - msg += f" in file {self.filename!r}" - if self.lineno is not None: - msg += f" on line {self.lineno}" - msg += ":\n" - msg += f"{self.message}\n" - return msg - @overload def warn_or_fail( @@ -215,33 +145,6 @@ def fail( warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) -def quoted_for_c_string(s: str) -> str: - for old, new in ( - ('\\', '\\\\'), # must be first! - ('"', '\\"'), - ("'", "\\'"), - ): - s = s.replace(old, new) - return s - -def c_repr(s: str) -> str: - return '"' + s + '"' - - -def wrapped_c_string_literal( - text: str, - *, - width: int = 72, - suffix: str = '', - initial_indent: int = 0, - subsequent_indent: int = 4 -) -> str: - wrapped = textwrap.wrap(text, width=width, replace_whitespace=False, - drop_whitespace=False, break_on_hyphens=False) - separator = '"' + suffix + '\n' + subsequent_indent * ' ' + '"' - return initial_indent * ' ' + '"' + separator.join(wrapped) + '"' - - is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match def is_legal_py_identifier(s: str) -> bool: @@ -266,20 +169,6 @@ def ensure_legal_c_identifier(s: str) -> str: return s + "_value" return s -def rstrip_lines(s: str) -> str: - text, add, output = _text_accumulator() - for line in s.split('\n'): - add(line.rstrip()) - add('\n') - text.pop() - return output() - -def format_escape(s: str) -> str: - # double up curly-braces, this string will be used - # as part of a format_map() template later - s = s.replace('{', '{{') - s = s.replace('}', '}}') - return s def linear_format(s: str, **kwargs: str) -> str: """ @@ -295,19 +184,16 @@ def linear_format(s: str, **kwargs: str) -> str: by the indent of the source line. * A newline will be added to the end. """ - - add, output = text_accumulator() + lines = [] for line in s.split('\n'): indent, curly, trailing = line.partition('{') if not curly: - add(line) - add('\n') + lines.extend([line, "\n"]) continue name, curly, trailing = trailing.partition('}') if not curly or name not in kwargs: - add(line) - add('\n') + lines.extend([line, "\n"]) continue if trailing: @@ -321,101 +207,11 @@ def linear_format(s: str, **kwargs: str) -> str: if not value: continue - value = textwrap.indent(rstrip_lines(value), indent) - add(value) - add('\n') - - return output()[:-1] - -def indent_all_lines(s: str, prefix: str) -> str: - """ - Returns 's', with 'prefix' prepended to all lines. - - If the last line is empty, prefix is not prepended - to it. (If s is blank, returns s unchanged.) - - (textwrap.indent only adds to non-blank lines.) - """ - split = s.split('\n') - last = split.pop() - final = [] - for line in split: - final.append(prefix) - final.append(line) - final.append('\n') - if last: - final.append(prefix) - final.append(last) - return ''.join(final) - -def suffix_all_lines(s: str, suffix: str) -> str: - """ - Returns 's', with 'suffix' appended to all lines. - - If the last line is empty, suffix is not appended - to it. (If s is blank, returns s unchanged.) - """ - split = s.split('\n') - last = split.pop() - final = [] - for line in split: - final.append(line) - final.append(suffix) - final.append('\n') - if last: - final.append(last) - final.append(suffix) - return ''.join(final) - - -def pprint_words(items: list[str]) -> str: - if len(items) <= 2: - return " and ".join(items) - else: - return ", ".join(items[:-1]) + " and " + items[-1] - + stripped = [line.rstrip() for line in value.split("\n")] + value = textwrap.indent("\n".join(stripped), indent) + lines.extend([value, "\n"]) -def version_splitter(s: str) -> tuple[int, ...]: - """Splits a version string into a tuple of integers. - - The following ASCII characters are allowed, and employ - the following conversions: - a -> -3 - b -> -2 - c -> -1 - (This permits Python-style version strings such as "1.4b3".) - """ - version: list[int] = [] - accumulator: list[str] = [] - def flush() -> None: - if not accumulator: - fail(f'Unsupported version string: {s!r}') - version.append(int(''.join(accumulator))) - accumulator.clear() - - for c in s: - if c.isdigit(): - accumulator.append(c) - elif c == '.': - flush() - elif c in 'abc': - flush() - version.append('abc'.index(c) - 3) - else: - fail(f'Illegal character {c!r} in version string {s!r}') - flush() - return tuple(version) - -def version_comparitor(version1: str, version2: str) -> Literal[-1, 0, 1]: - iterator = itertools.zip_longest( - version_splitter(version1), version_splitter(version2), fillvalue=0 - ) - for a, b in iterator: - if a < b: - return -1 - if a > b: - return 1 - return 0 + return "".join(lines[:-1]) class CRenderData: @@ -653,34 +449,6 @@ def permute_optional_groups( return tuple(accumulator) -def strip_leading_and_trailing_blank_lines(s: str) -> str: - lines = s.rstrip().split('\n') - while lines: - line = lines[0] - if line.strip(): - break - del lines[0] - return '\n'.join(lines) - -@functools.lru_cache() -def normalize_snippet( - s: str, - *, - indent: int = 0 -) -> str: - """ - Reformats s: - * removes leading and trailing blank lines - * ensures that it does not end with a newline - * dedents so the first nonwhite character on any line is at column "indent" - """ - s = strip_leading_and_trailing_blank_lines(s) - s = textwrap.dedent(s) - if indent: - s = textwrap.indent(s, ' ' * indent) - return s - - def declare_parser( f: Function, *, @@ -751,62 +519,7 @@ def declare_parser( }}; #undef KWTUPLE """ % (format_ or fname) - return normalize_snippet(declarations) - - -def wrap_declarations( - text: str, - length: int = 78 -) -> str: - """ - A simple-minded text wrapper for C function declarations. - - It views a declaration line as looking like this: - xxxxxxxx(xxxxxxxxx,xxxxxxxxx) - If called with length=30, it would wrap that line into - xxxxxxxx(xxxxxxxxx, - xxxxxxxxx) - (If the declaration has zero or one parameters, this - function won't wrap it.) - - If this doesn't work properly, it's probably better to - start from scratch with a more sophisticated algorithm, - rather than try and improve/debug this dumb little function. - """ - lines = [] - for line in text.split('\n'): - prefix, _, after_l_paren = line.partition('(') - if not after_l_paren: - lines.append(line) - continue - in_paren, _, after_r_paren = after_l_paren.partition(')') - if not _: - lines.append(line) - continue - if ',' not in in_paren: - lines.append(line) - continue - parameters = [x.strip() + ", " for x in in_paren.split(',')] - prefix += "(" - if len(prefix) < length: - spaces = " " * len(prefix) - else: - spaces = " " * 4 - - while parameters: - line = prefix - first = True - while parameters: - if (not first and - (len(line) + len(parameters[0]) > length)): - break - line += parameters.pop(0) - first = False - if not parameters: - line = line.rstrip(", ") + ")" + after_r_paren - lines.append(line.rstrip()) - prefix = spaces - return "\n".join(lines) + return libclinic.normalize_snippet(declarations) class CLanguage(Language): @@ -818,54 +531,95 @@ class CLanguage(Language): stop_line = "[{dsl_name} start generated code]*/" checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" - PARSER_PROTOTYPE_KEYWORD: Final[str] = normalize_snippet(""" + NO_VARARG: Final[str] = "PY_SSIZE_T_MAX" + + PARSER_PROTOTYPE_KEYWORD: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) """) - PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = libclinic.normalize_snippet(""" static int {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) """) - PARSER_PROTOTYPE_VARARGS: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_VARARGS: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *args) """) - PARSER_PROTOTYPE_FASTCALL: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_FASTCALL: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) """) - PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) """) - PARSER_PROTOTYPE_DEF_CLASS: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_DEF_CLASS: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) """) - PARSER_PROTOTYPE_NOARGS: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_NOARGS: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) """) - METH_O_PROTOTYPE: Final[str] = normalize_snippet(""" + PARSER_PROTOTYPE_GETTER: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, void *Py_UNUSED(context)) + """) + PARSER_PROTOTYPE_SETTER: Final[str] = libclinic.normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *value, void *Py_UNUSED(context)) + """) + METH_O_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" static PyObject * {c_basename}({impl_parameters}) """) - DOCSTRING_PROTOTYPE_VAR: Final[str] = normalize_snippet(""" + DOCSTRING_PROTOTYPE_VAR: Final[str] = libclinic.normalize_snippet(""" PyDoc_VAR({c_basename}__doc__); """) - DOCSTRING_PROTOTYPE_STRVAR: Final[str] = normalize_snippet(""" + DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" PyDoc_STRVAR({c_basename}__doc__, {docstring}); """) - IMPL_DEFINITION_PROTOTYPE: Final[str] = normalize_snippet(""" + GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_STRVAR({getset_basename}__doc__, + {docstring}); + #define {getset_basename}_HAS_DOCSTR + """) + IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" static {impl_return_type} {c_basename}_impl({impl_parameters}) """) - METHODDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" + METHODDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" #define {methoddef_name} \ {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, """) - METHODDEF_PROTOTYPE_IFNDEF: Final[str] = normalize_snippet(""" + GETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #if defined({getset_basename}_HAS_DOCSTR) + # define {getset_basename}_DOCSTR {getset_basename}__doc__ + #else + # define {getset_basename}_DOCSTR NULL + #endif + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, NULL, {getset_basename}_DOCSTR}}, + #endif + """) + SETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #if defined({getset_name}_HAS_DOCSTR) + # define {getset_basename}_DOCSTR {getset_basename}__doc__ + #else + # define {getset_basename}_DOCSTR NULL + #endif + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", NULL, (setter){getset_basename}_set, NULL}}, + #endif + """) + METHODDEF_PROTOTYPE_IFNDEF: Final[str] = libclinic.normalize_snippet(""" #ifndef {methoddef_name} #define {methoddef_name} #endif /* !defined({methoddef_name}) */ @@ -894,8 +648,7 @@ class CLanguage(Language): def __init__(self, filename: str) -> None: super().__init__(filename) - self.cpp = cpp.Monitor(filename) - self.cpp.fail = fail # type: ignore[method-assign] + self.cpp = libclinic.cpp.Monitor(filename) def parse_line(self, line: str) -> None: self.cpp.writeline(line) @@ -932,9 +685,9 @@ def compiler_deprecated_warning( code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( major=minversion[0], minor=minversion[1], - message=c_repr(message), + message=libclinic.c_repr(message), ) - return normalize_snippet(code) + return libclinic.normalize_snippet(code) def deprecate_positional_use( self, @@ -963,7 +716,7 @@ def deprecate_positional_use( params.values(), key=attrgetter("deprecated_positional") ): names = [repr(p.name) for p in group] - pstr = pprint_words(names) + pstr = libclinic.pprint_words(names) if len(names) == 1: message += ( f" Parameter {pstr} will become a keyword-only parameter " @@ -982,10 +735,10 @@ def deprecate_positional_use( code = self.DEPRECATION_WARNING_PROTOTYPE.format( condition=condition, errcheck="", - message=wrapped_c_string_literal(message, width=64, - subsequent_indent=20), + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), ) - return normalize_snippet(code, indent=4) + return libclinic.normalize_snippet(code, indent=4) def deprecate_keyword_use( self, @@ -1032,7 +785,7 @@ def deprecate_keyword_use( else: condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" names = [repr(p.name) for p in params.values()] - pstr = pprint_words(names) + pstr = libclinic.pprint_words(names) pl = 's' if len(params) != 1 else '' message = ( f"Passing keyword argument{pl} {pstr} to " @@ -1043,7 +796,7 @@ def deprecate_keyword_use( params.values(), key=attrgetter("deprecated_keyword") ): names = [repr(p.name) for p in group] - pstr = pprint_words(names) + pstr = libclinic.pprint_words(names) pl = 's' if len(names) != 1 else '' message += ( f" Parameter{pl} {pstr} will become positional-only " @@ -1065,30 +818,10 @@ def deprecate_keyword_use( code = self.DEPRECATION_WARNING_PROTOTYPE.format( condition=condition, errcheck=errcheck, - message=wrapped_c_string_literal(message, width=64, - subsequent_indent=20), + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), ) - return normalize_snippet(code, indent=4) - - def docstring_for_c_string( - self, - f: Function - ) -> str: - text, add, output = _text_accumulator() - # turn docstring into a properly quoted C string - for line in f.docstring.split('\n'): - add('"') - add(quoted_for_c_string(line)) - add('\\n"\n') - - if text[-2] == sig_end_marker: - # If we only have a signature, add the blank line that the - # __text_signature__ getter expects to be there. - add('"\\n"') - else: - text.pop() - add('"') - return ''.join(text) + return libclinic.normalize_snippet(code, indent=4) def output_templates( self, @@ -1118,7 +851,7 @@ def output_templates( and not f.critical_section) new_or_init = f.kind.new_or_init - vararg: int | str = NO_VARARG + vararg: int | str = self.NO_VARARG pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 for i, p in enumerate(parameters, 1): if p.is_keyword_only(): @@ -1126,12 +859,12 @@ def output_templates( if not p.is_optional(): min_kw_only = i - max_pos elif p.is_vararg(): - if vararg != NO_VARARG: + if vararg != self.NO_VARARG: fail("Too many var args") pseudo_args += 1 vararg = i - 1 else: - if vararg == NO_VARARG: + if vararg == self.NO_VARARG: max_pos = i if p.is_positional_only(): pos_only = i @@ -1161,6 +894,19 @@ def output_templates( methoddef_define = self.METHODDEF_PROTOTYPE_DEFINE if new_or_init and not f.docstring: docstring_prototype = docstring_definition = '' + elif f.kind is GETTER: + methoddef_define = self.GETTERDEF_PROTOTYPE_DEFINE + if f.docstring: + docstring_prototype = '' + docstring_definition = self.GETSET_DOCSTRING_PROTOTYPE_STRVAR + else: + docstring_prototype = docstring_definition = '' + elif f.kind is SETTER: + if f.docstring: + fail("docstrings are only supported for @getter, not @setter") + return_value_declaration = "int {return_value};" + methoddef_define = self.SETTERDEF_PROTOTYPE_DEFINE + docstring_prototype = docstring_definition = '' else: docstring_prototype = self.DOCSTRING_PROTOTYPE_VAR docstring_definition = self.DOCSTRING_PROTOTYPE_STRVAR @@ -1176,18 +922,18 @@ def parser_body( declarations: str = '' ) -> str: nonlocal parser_body_fields - add, output = text_accumulator() - add(prototype) + lines = [] + lines.append(prototype) parser_body_fields = fields - preamble = normalize_snippet(""" + preamble = libclinic.normalize_snippet(""" {{ {return_value_declaration} {parser_declarations} {declarations} {initializers} """) + "\n" - finale = normalize_snippet(""" + finale = libclinic.normalize_snippet(""" {modifications} {lock} {return_value} = {c_basename}_impl({impl_arguments}); @@ -1201,9 +947,8 @@ def parser_body( }} """) for field in preamble, *fields, finale: - add('\n') - add(field) - return linear_format(output(), parser_declarations=declarations) + lines.append(field) + return linear_format("\n".join(lines), parser_declarations=declarations) fastcall = not new_or_init limited_capi = clinic.limited_capi @@ -1215,9 +960,20 @@ def parser_body( limited_capi = False parsearg: str | None + if f.kind in {GETTER, SETTER} and parameters: + fail(f"@{f.kind.name.lower()} method cannot define parameters") + if not parameters: parser_code: list[str] | None - if not requires_defining_class: + if f.kind is GETTER: + flags = "" # This should end up unused + parser_prototype = self.PARSER_PROTOTYPE_GETTER + parser_code = [] + elif f.kind is SETTER: + flags = "" + parser_prototype = self.PARSER_PROTOTYPE_SETTER + parser_code = [] + elif not requires_defining_class: # no parameters, METH_NOARGS flags = "METH_NOARGS" parser_prototype = self.PARSER_PROTOTYPE_NOARGS @@ -1229,7 +985,7 @@ def parser_body( parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS return_error = ('return NULL;' if simple_return else 'goto exit;') - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (nargs) {{ PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); %s @@ -1269,7 +1025,7 @@ def parser_body( argname = 'arg' if parameters[0].name == argname: argname += '_' - parser_prototype = normalize_snippet(""" + parser_prototype = libclinic.normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *%s) """ % argname) @@ -1283,7 +1039,7 @@ def parser_body( }} """ % argname parser_definition = parser_body(parser_prototype, - normalize_snippet(parsearg, indent=4)) + libclinic.normalize_snippet(parsearg, indent=4)) elif has_option_groups: # positional parameters with option groups @@ -1317,15 +1073,16 @@ def parser_body( argname_fmt = 'PyTuple_GET_ITEM(args, %d)' left_args = f"{nargs} - {max_pos}" - max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos + max_args = self.NO_VARARG if (vararg != self.NO_VARARG) else max_pos if limited_capi: parser_code = [] if nargs != 'nargs': - parser_code.append(normalize_snippet(f'Py_ssize_t nargs = {nargs};', indent=4)) + nargs_def = f'Py_ssize_t nargs = {nargs};' + parser_code.append(libclinic.normalize_snippet(nargs_def, indent=4)) nargs = 'nargs' if min_pos == max_args: pl = '' if min_pos == 1 else 's' - parser_code.append(normalize_snippet(f""" + parser_code.append(libclinic.normalize_snippet(f""" if ({nargs} != {min_pos}) {{{{ PyErr_Format(PyExc_TypeError, "{{name}} expected {min_pos} argument{pl}, got %zd", {nargs}); goto exit; @@ -1335,16 +1092,16 @@ def parser_body( else: if min_pos: pl = '' if min_pos == 1 else 's' - parser_code.append(normalize_snippet(f""" + parser_code.append(libclinic.normalize_snippet(f""" if ({nargs} < {min_pos}) {{{{ PyErr_Format(PyExc_TypeError, "{{name}} expected at least {min_pos} argument{pl}, got %zd", {nargs}); goto exit; }}}} """, indent=4)) - if max_args != NO_VARARG: + if max_args != self.NO_VARARG: pl = '' if max_args == 1 else 's' - parser_code.append(normalize_snippet(f""" + parser_code.append(libclinic.normalize_snippet(f""" if ({nargs} > {max_args}) {{{{ PyErr_Format(PyExc_TypeError, "{{name}} expected at most {max_args} argument{pl}, got %zd", {nargs}); goto exit; @@ -1354,7 +1111,7 @@ def parser_body( else: clinic.add_include('pycore_modsupport.h', '_PyArg_CheckPositional()') - parser_code = [normalize_snippet(f""" + parser_code = [libclinic.normalize_snippet(f""" if (!_PyArg_CheckPositional("{{name}}", {nargs}, {min_pos}, {max_args})) {{{{ goto exit; }}}} @@ -1364,7 +1121,7 @@ def parser_body( for i, p in enumerate(parameters): if p.is_vararg(): if fastcall: - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" %s = PyTuple_New(%s); if (!%s) {{ goto exit; @@ -1381,7 +1138,7 @@ def parser_body( max_pos ), indent=4)) else: - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" %s = PyTuple_GetSlice(%d, -1); """ % ( p.converter.parser_name, @@ -1397,12 +1154,12 @@ def parser_body( break if has_optional or p.is_optional(): has_optional = True - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" if (%s < %d) {{ goto skip_optional; }} """, indent=4) % (nargs, i + 1)) - parser_code.append(normalize_snippet(parsearg, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) if parser_code is not None: if has_optional: @@ -1413,7 +1170,7 @@ def parser_body( if fastcall: clinic.add_include('pycore_modsupport.h', '_PyArg_ParseStack()') - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", {parse_arguments})) {{ goto exit; @@ -1422,7 +1179,7 @@ def parser_body( else: flags = "METH_VARARGS" parser_prototype = self.PARSER_PROTOTYPE_VARARGS - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ goto exit; @@ -1439,13 +1196,16 @@ def parser_body( if p.deprecated_keyword: deprecated_keywords[i] = p - has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) + has_optional_kw = ( + max(pos_only, min_pos) + min_kw_only + < len(converters) - int(vararg != self.NO_VARARG) + ) if limited_capi: parser_code = None fastcall = False else: - if vararg == NO_VARARG: + if vararg == self.NO_VARARG: clinic.add_include('pycore_modsupport.h', '_PyArg_UnpackKeywords()') args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( @@ -1474,7 +1234,7 @@ def parser_body( declarations += "\nPyObject *argsbuf[%s];" % len(converters) if has_optional_kw: declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); if (!args) {{ goto exit; @@ -1492,7 +1252,7 @@ def parser_body( declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" if has_optional_kw: declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); if (!fastargs) {{ goto exit; @@ -1525,19 +1285,19 @@ def parser_body( parser_code.append("%s:" % add_label) add_label = None if not p.is_optional(): - parser_code.append(normalize_snippet(parsearg, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) elif i < pos_only: add_label = 'skip_optional_posonly' - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" if (nargs < %d) {{ goto %s; }} """ % (i + 1, add_label), indent=4)) if has_optional_kw: - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" noptargs--; """, indent=4)) - parser_code.append(normalize_snippet(parsearg, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) else: if i < max_pos: label = 'skip_optional_pos' @@ -1545,24 +1305,24 @@ def parser_body( else: label = 'skip_optional_kwonly' first_opt = max_pos + min_kw_only - if vararg != NO_VARARG: + if vararg != self.NO_VARARG: first_opt += 1 if i == first_opt: add_label = label - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" if (!noptargs) {{ goto %s; }} """ % add_label, indent=4)) if i + 1 == len(parameters): - parser_code.append(normalize_snippet(parsearg, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) else: add_label = label - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(""" if (%s) {{ """ % (argname_fmt % i), indent=4)) - parser_code.append(normalize_snippet(parsearg, indent=8)) - parser_code.append(normalize_snippet(""" + parser_code.append(libclinic.normalize_snippet(parsearg, indent=8)) + parser_code.append(libclinic.normalize_snippet(""" if (!--noptargs) {{ goto %s; }} @@ -1581,7 +1341,7 @@ def parser_body( assert not fastcall flags = "METH_VARARGS|METH_KEYWORDS" parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, {parse_arguments})) goto exit; @@ -1593,7 +1353,7 @@ def parser_body( elif fastcall: clinic.add_include('pycore_modsupport.h', '_PyArg_ParseStackAndKeywords()') - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} {parse_arguments})) {{ goto exit; @@ -1602,7 +1362,7 @@ def parser_body( else: clinic.add_include('pycore_modsupport.h', '_PyArg_ParseTupleAndKeywordsFast()') - parser_code = [normalize_snippet(""" + parser_code = [libclinic.normalize_snippet(""" if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, {parse_arguments})) {{ goto exit; @@ -1649,7 +1409,7 @@ def parser_body( declarations = '{base_type_ptr}' clinic.add_include('pycore_modsupport.h', '_PyArg_NoKeywords()') - fields.insert(0, normalize_snippet(""" + fields.insert(0, libclinic.normalize_snippet(""" if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ goto exit; }} @@ -1657,7 +1417,7 @@ def parser_body( if not parses_positional: clinic.add_include('pycore_modsupport.h', '_PyArg_NoPositional()') - fields.insert(0, normalize_snippet(""" + fields.insert(0, libclinic.normalize_snippet(""" if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ goto exit; }} @@ -1670,6 +1430,8 @@ def parser_body( methoddef_cast_end = "" if flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): methoddef_cast = "(PyCFunction)" + elif f.kind is GETTER: + methoddef_cast = "" # This should end up unused elif limited_capi: methoddef_cast = "(PyCFunction)(void(*)(void))" else: @@ -1765,7 +1527,7 @@ def render_option_group_parsing( # Clinic prefers groups on the left. So in the above example, # five arguments would map to B+C, not C+D. - add, output = text_accumulator() + out = [] parameters = list(f.parameters.values()) if isinstance(parameters[0].converter, self_converter): del parameters[0] @@ -1797,14 +1559,14 @@ def render_option_group_parsing( nargs = 'PyTuple_Size(args)' else: nargs = 'PyTuple_GET_SIZE(args)' - add(f"switch ({nargs}) {{\n") + out.append(f"switch ({nargs}) {{\n") for subset in permute_optional_groups(left, required, right): count = len(subset) count_min = min(count_min, count) count_max = max(count_max, count) if count == 0: - add(""" case 0: + out.append(""" case 0: break; """) continue @@ -1836,14 +1598,15 @@ def render_option_group_parsing( """ s = linear_format(s, group_booleans=lines) s = s.format_map(d) - add(s) + out.append(s) - add(" default:\n") + out.append(" default:\n") s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' - add(s.format(f.full_name, count_min, count_max)) - add(' goto exit;\n') - add("}") - template_dict['option_group_parsing'] = format_escape(output()) + out.append(s.format(f.full_name, count_min, count_max)) + out.append(' goto exit;\n') + out.append("}") + + template_dict['option_group_parsing'] = libclinic.format_escape("".join(out)) def render_function( self, @@ -1853,7 +1616,6 @@ def render_function( if f is None or clinic is None: return "" - add, output = text_accumulator() data = CRenderData() assert f.parameters, "We should always have a 'self' at this point!" @@ -1927,20 +1689,34 @@ def render_function( full_name = f.full_name template_dict = {'full_name': full_name} template_dict['name'] = f.displayname - template_dict['c_basename'] = f.c_basename - template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" - - template_dict['docstring'] = self.docstring_for_c_string(f) + if f.kind in {GETTER, SETTER}: + template_dict['getset_name'] = f.c_basename.upper() + template_dict['getset_basename'] = f.c_basename + if f.kind is GETTER: + template_dict['c_basename'] = f.c_basename + "_get" + elif f.kind is SETTER: + template_dict['c_basename'] = f.c_basename + "_set" + # Implicitly add the setter value parameter. + data.impl_parameters.append("PyObject *value") + data.impl_arguments.append("value") + else: + template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" + template_dict['c_basename'] = f.c_basename + template_dict['docstring'] = libclinic.docstring_for_c_string(f.docstring) template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' template_dict['target_critical_section'] = ', '.join(f.target_critical_section) for converter in converters: converter.set_template_dict(template_dict) f.return_converter.render(f, data) - template_dict['impl_return_type'] = f.return_converter.type + if f.kind is SETTER: + # All setters return an int. + template_dict['impl_return_type'] = 'int' + else: + template_dict['impl_return_type'] = f.return_converter.type - template_dict['declarations'] = format_escape("\n".join(data.declarations)) + template_dict['declarations'] = libclinic.format_escape("\n".join(data.declarations)) template_dict['initializers'] = "\n\n".join(data.initializers) template_dict['modifications'] = '\n\n'.join(data.modifications) template_dict['keywords_c'] = ' '.join('"' + k + '",' @@ -1956,9 +1732,11 @@ def render_function( template_dict['parse_arguments_comma'] = ''; template_dict['impl_parameters'] = ", ".join(data.impl_parameters) template_dict['impl_arguments'] = ", ".join(data.impl_arguments) - template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip()) - template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip()) - template_dict['cleanup'] = format_escape("".join(data.cleanup)) + + template_dict['return_conversion'] = libclinic.format_escape("".join(data.return_conversion).rstrip()) + template_dict['post_parsing'] = libclinic.format_escape("".join(data.post_parsing).rstrip()) + template_dict['cleanup'] = libclinic.format_escape("".join(data.cleanup)) + template_dict['return_value'] = data.return_value template_dict['lock'] = "\n".join(data.lock) template_dict['unlock'] = "\n".join(data.unlock) @@ -2002,12 +1780,12 @@ def render_function( # mild hack: # reflow long impl declarations if name in {"impl_prototype", "impl_definition"}: - s = wrap_declarations(s) + s = libclinic.wrap_declarations(s) if clinic.line_prefix: - s = indent_all_lines(s, clinic.line_prefix) + s = libclinic.indent_all_lines(s, clinic.line_prefix) if clinic.line_suffix: - s = suffix_all_lines(s, clinic.line_suffix) + s = libclinic.suffix_all_lines(s, clinic.line_suffix) destination.append(s) @@ -2167,7 +1945,7 @@ def _line(self, lookahead: bool = False) -> str: return line def parse_verbatim_block(self) -> Block: - add, output = text_accumulator() + lines = [] self.block_start_line_number = self.line_number while self.input: @@ -2176,12 +1954,12 @@ def parse_verbatim_block(self) -> Block: if dsl_name: self.dsl_name = dsl_name break - add(line) + lines.append(line) - return Block(output()) + return Block("".join(lines)) def parse_clinic_block(self, dsl_name: str) -> Block: - input_add, input_output = text_accumulator() + in_lines = [] self.block_start_line_number = self.line_number + 1 stop_line = self.language.stop_line.format(dsl_name=dsl_name) body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) @@ -2209,7 +1987,7 @@ def is_stop_line(line: str) -> bool: line = line.lstrip() assert line.startswith(body_prefix) line = line.removeprefix(body_prefix) - input_add(line) + in_lines.append(line) # consume output and checksum line, if present. if self.last_dsl_name == dsl_name: @@ -2223,7 +2001,7 @@ def is_stop_line(line: str) -> bool: assert checksum_re is not None # scan forward for checksum line - output_add, output_output = text_accumulator() + out_lines = [] arguments = None while self.input: line = self._line(lookahead=True) @@ -2231,12 +2009,12 @@ def is_stop_line(line: str) -> bool: arguments = match.group(1) if match else None if arguments: break - output_add(line) + out_lines.append(line) if self.is_start_line(line): break output: str | None - output = output_output() + output = "".join(out_lines) if arguments: d = {} for field in shlex.split(arguments): @@ -2264,7 +2042,7 @@ def is_stop_line(line: str) -> bool: self.input.extend(reversed(output_lines)) output = None - return Block(input_output(), dsl_name, output=output) + return Block("".join(in_lines), dsl_name, output=output) @dc.dataclass(slots=True, frozen=True) @@ -2292,6 +2070,9 @@ class BlockPrinter: language: Language f: io.StringIO = dc.field(default_factory=io.StringIO) + # '#include "header.h" // reason': column of '//' comment + INCLUDE_COMMENT_COLUMN: Final[int] = 35 + def print_block( self, block: Block, @@ -2347,7 +2128,7 @@ def print_block( line = f'#include "{include.filename}"' if include.reason: comment = f'// {include.reason}\n' - line = line.ljust(INCLUDE_COMMENT_COLUMN - 1) + comment + line = line.ljust(self.INCLUDE_COMMENT_COLUMN - 1) + comment output += line if current_condition: @@ -2384,26 +2165,26 @@ class BufferSeries: def __init__(self) -> None: self._start = 0 - self._array: list[_TextAccumulator] = [] - self._constructor = _text_accumulator + self._array: list[list[str]] = [] - def __getitem__(self, i: int) -> _TextAccumulator: + def __getitem__(self, i: int) -> list[str]: i -= self._start if i < 0: self._start += i - prefix = [self._constructor() for x in range(-i)] + prefix: list[list[str]] = [[] for x in range(-i)] self._array = prefix + self._array i = 0 while i >= len(self._array): - self._array.append(self._constructor()) + self._array.append([]) return self._array[i] def clear(self) -> None: for ta in self._array: - ta.text.clear() + ta.clear() def dump(self) -> str: - texts = [ta.output() for ta in self._array] + texts = ["".join(ta) for ta in self._array] + self.clear() return "".join(texts) @@ -2589,7 +2370,7 @@ def __init__( 'impl_definition': d('block'), } - DestBufferType = dict[str, _TextAccumulator] + DestBufferType = dict[str, list[str]] DestBufferList = list[DestBufferType] self.destination_buffers_stack: DestBufferList = [] @@ -2661,7 +2442,7 @@ def get_destination_buffer( self, name: str, item: int = 0 - ) -> _TextAccumulator: + ) -> list[str]: d = self.get_destination(name) return d.buffers[item] @@ -2932,6 +2713,8 @@ class FunctionKind(enum.Enum): CLASS_METHOD = enum.auto() METHOD_INIT = enum.auto() METHOD_NEW = enum.auto() + GETTER = enum.auto() + SETTER = enum.auto() @functools.cached_property def new_or_init(self) -> bool: @@ -2947,6 +2730,8 @@ def __repr__(self) -> str: CLASS_METHOD: Final = FunctionKind.CLASS_METHOD METHOD_INIT: Final = FunctionKind.METHOD_INIT METHOD_NEW: Final = FunctionKind.METHOD_NEW +GETTER: Final = FunctionKind.GETTER +SETTER: Final = FunctionKind.SETTER ParamDict = dict[str, "Parameter"] ReturnConverterType = Callable[..., "CReturnConverter"] @@ -3033,7 +2818,8 @@ def methoddef_flags(self) -> str | None: case FunctionKind.STATIC_METHOD: flags.append('METH_STATIC') case _ as kind: - assert kind is FunctionKind.CALLABLE, f"unknown kind: {kind!r}" + acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER, FunctionKind.SETTER} + assert kind in acceptable_kinds, f"unknown kind: {kind!r}" if self.coexist: flags.append('METH_COEXIST') return '|'.join(flags) @@ -3110,11 +2896,9 @@ def get_displayname(self, i: int) -> str: return f'argument {i}' def render_docstring(self) -> str: - add, out = text_accumulator() - add(f" {self.name}\n") - for line in self.docstring.split("\n"): - add(f" {line}\n") - return out().rstrip() + lines = [f" {self.name}"] + lines.extend(f" {line}" for line in self.docstring.split("\n")) + return "\n".join(lines).rstrip() CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) @@ -3160,7 +2944,7 @@ def closure(f: CConverterClassT) -> CConverterClassT: class CConverterAutoRegister(type): def __init__( - cls, name: str, bases: tuple[type, ...], classdict: dict[str, Any] + cls, name: str, bases: tuple[type[object], ...], classdict: dict[str, Any] ) -> None: converter_cls = cast(type["CConverter"], cls) add_c_converter(converter_cls) @@ -3193,7 +2977,7 @@ class CConverter(metaclass=CConverterAutoRegister): # If not None, default must be isinstance() of this type. # (You can also specify a tuple of types.) - default_type: bltns.type[Any] | tuple[bltns.type[Any], ...] | None = None + default_type: bltns.type[object] | tuple[bltns.type[object], ...] | None = None # "default" converted into a C value, as a string. # Or None if there is no default. @@ -3432,7 +3216,7 @@ def parse_argument(self, args: list[str]) -> None: args.append(self.converter) if self.encoding: - args.append(c_repr(self.encoding)) + args.append(libclinic.c_repr(self.encoding)) elif self.subclass_of: args.append(self.subclass_of) @@ -3610,8 +3394,8 @@ def set_template_dict(self, template_dict: TemplateDict) -> None: @property def parser_name(self) -> str: - if self.name in CLINIC_PREFIXED_ARGS: # bpo-39741 - return CLINIC_PREFIX + self.name + if self.name in libclinic.CLINIC_PREFIXED_ARGS: # bpo-39741 + return libclinic.CLINIC_PREFIX + self.name else: return self.name @@ -3659,7 +3443,7 @@ def add_include(self, name: str, reason: str, ReturnConverterDict = dict[str, ReturnConverterType] return_converters: ReturnConverterDict = {} -TypeSet = set[bltns.type[Any]] +TypeSet = set[bltns.type[object]] class bool_converter(CConverter): @@ -3673,7 +3457,7 @@ def converter_init(self, *, accept: TypeSet = {object}) -> None: self.format_unit = 'i' elif accept != {object}: fail(f"bool_converter: illegal 'accept' argument {accept!r}") - if self.default is not unspecified: + if self.default is not unspecified and self.default is not unknown: self.default = bool(self.default) self.c_default = str(int(self.default)) @@ -4323,7 +4107,7 @@ class buffer: pass class rwbuffer: pass class robuffer: pass -StrConverterKeyType = tuple[frozenset[type], bool, bool] +StrConverterKeyType = tuple[frozenset[type[object]], bool, bool] def str_converter_key( types: TypeSet, encoding: bool | str | None, zeroes: bool @@ -4678,7 +4462,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st def correct_name_for_self( f: Function ) -> tuple[str, str]: - if f.kind in (CALLABLE, METHOD_INIT): + if f.kind in {CALLABLE, METHOD_INIT, GETTER, SETTER}: if f.cls: return "PyObject *", "self" return "PyObject *", "module" @@ -4822,7 +4606,7 @@ class CReturnConverterAutoRegister(type): def __init__( cls: ReturnConverterType, name: str, - bases: tuple[type, ...], + bases: tuple[type[object], ...], classdict: dict[str, Any] ) -> None: add_c_return_converter(cls) @@ -5177,13 +4961,6 @@ def reset(self) -> None: self.critical_section = False self.target_critical_section = [] - def directive_version(self, required: str) -> None: - global version - if version_comparitor(version, required) < 0: - fail("Insufficient Clinic version!\n" - f" Version: {version}\n" - f" Required: {required}") - def directive_module(self, name: str) -> None: fields = name.split('.')[:-1] module, cls = self.clinic._module_and_class(fields) @@ -5310,6 +5087,24 @@ def at_critical_section(self, *args: str) -> None: self.target_critical_section.extend(args) self.critical_section = True + def at_getter(self) -> None: + match self.kind: + case FunctionKind.GETTER: + fail("Cannot apply @getter twice to the same function!") + case FunctionKind.SETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.GETTER + + def at_setter(self) -> None: + match self.kind: + case FunctionKind.SETTER: + fail("Cannot apply @setter twice to the same function!") + case FunctionKind.GETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.SETTER + def at_staticmethod(self) -> None: if self.kind is not CALLABLE: fail("Can't set @staticmethod, function is not a normal callable") @@ -5419,14 +5214,20 @@ def update_function_kind(self, fullname: str) -> None: _, cls = self.clinic._module_and_class(fields) if name in unsupported_special_methods: fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") + if name == '__new__': - if (self.kind is not CLASS_METHOD) or (not cls): + if (self.kind is CLASS_METHOD) and cls: + self.kind = METHOD_NEW + else: fail("'__new__' must be a class method!") - self.kind = METHOD_NEW elif name == '__init__': - if (self.kind is not CALLABLE) or (not cls): - fail("'__init__' must be a normal method, not a class or static method!") - self.kind = METHOD_INIT + if (self.kind is CALLABLE) and cls: + self.kind = METHOD_INIT + else: + fail( + "'__init__' must be a normal method; " + f"got '{self.kind}'!" + ) def state_modulename_name(self, line: str) -> None: # looking for declaration, which establishes the leftmost column @@ -5503,6 +5304,8 @@ def state_modulename_name(self, line: str) -> None: return_converter = None if returns: + if self.kind in {GETTER, SETTER}: + fail(f"@{self.kind.name.lower()} method cannot define a return type") ast_input = f"def x() -> {returns}: pass" try: module_node = ast.parse(ast_input) @@ -5524,6 +5327,10 @@ def state_modulename_name(self, line: str) -> None: function_name = fields.pop() module, cls = self.clinic._module_and_class(fields) + if self.kind in {GETTER, SETTER}: + if not cls: + fail("@getter and @setter must be methods") + self.update_function_kind(full_name) if self.kind is METHOD_INIT and not return_converter: return_converter = init_return_converter() @@ -5752,11 +5559,11 @@ def parse_parameter(self, line: str) -> None: parameter_name = parameter.arg name, legacy, kwargs = self.parse_converter(parameter.annotation) + value: object if not default: if self.parameter_state is ParamState.OPTIONAL: fail(f"Can't have a parameter without a default ({parameter_name!r}) " "after a parameter with a default!") - value: Sentinels | Null if is_vararg: value = NULL kwargs.setdefault('c_default', "NULL") @@ -5870,7 +5677,7 @@ def bad_node(self, node: ast.AST) -> None: if isinstance(value, (bool, NoneType)): c_default = "Py_" + py_default elif isinstance(value, str): - c_default = c_repr(value) + c_default = libclinic.c_repr(value) else: c_default = py_default @@ -6157,12 +5964,15 @@ def state_function_docstring(self, line: str) -> None: def format_docstring_signature( self, f: Function, parameters: list[Parameter] ) -> str: - text, add, output = _text_accumulator() - add(f.displayname) + lines = [] + lines.append(f.displayname) if self.forced_text_signature: - add(self.forced_text_signature) + lines.append(self.forced_text_signature) + elif f.kind in {GETTER, SETTER}: + # @getter and @setter do not need signatures like a method or a function. + return '' else: - add('(') + lines.append('(') # populate "right_bracket_count" field for every parameter assert parameters, "We should always have a self parameter. " + repr(f) @@ -6216,7 +6026,7 @@ def fix_right_bracket_count(desired: int) -> str: first_parameter = True last_p = parameters[-1] - line_length = len(''.join(text)) + line_length = len(''.join(lines)) indent = " " * line_length def add_parameter(text: str) -> None: nonlocal line_length @@ -6227,12 +6037,11 @@ def add_parameter(text: str) -> None: else: s = ' ' + text if line_length + len(s) >= 72: - add('\n') - add(indent) + lines.extend(["\n", indent]) line_length = len(indent) s = text line_length += len(s) - add(s) + lines.append(s) for p in parameters: if not p.converter.show_in_signature: @@ -6255,8 +6064,7 @@ def add_parameter(text: str) -> None: added_star = True add_parameter('*,') - p_add, p_output = text_accumulator() - p_add(fix_right_bracket_count(p.right_bracket_count)) + p_lines = [fix_right_bracket_count(p.right_bracket_count)] if isinstance(p.converter, self_converter): # annotate first parameter as being a "self". @@ -6274,30 +6082,31 @@ def add_parameter(text: str) -> None: # have a docstring.) if this is an __init__ # (or __new__), then this signature is for # calling the class to construct a new instance. - p_add('$') + p_lines.append('$') if p.is_vararg(): - p_add("*") + p_lines.append("*") name = p.converter.signature_name or p.name - p_add(name) + p_lines.append(name) if not p.is_vararg() and p.converter.is_optional(): - p_add('=') + p_lines.append('=') value = p.converter.py_default if not value: value = repr(p.converter.default) - p_add(value) + p_lines.append(value) if (p != last_p) or need_a_trailing_slash: - p_add(',') + p_lines.append(',') - add_parameter(p_output()) + p_output = "".join(p_lines) + add_parameter(p_output) - add(fix_right_bracket_count(0)) + lines.append(fix_right_bracket_count(0)) if need_a_trailing_slash: add_parameter('/') - add(')') + lines.append(')') # PEP 8 says: # @@ -6309,13 +6118,13 @@ def add_parameter(text: str) -> None: # therefore this is commented out: # # if f.return_converter.py_default: - # add(' -> ') - # add(f.return_converter.py_default) + # lines.append(' -> ') + # lines.append(f.return_converter.py_default) if not f.docstring_only: - add("\n" + sig_end_marker + "\n") + lines.append("\n" + libclinic.SIG_END_MARKER + "\n") - signature_line = output() + signature_line = "".join(lines) # now fix up the places where the brackets look wrong return signature_line.replace(', ]', ',] ') @@ -6323,18 +6132,13 @@ def add_parameter(text: str) -> None: @staticmethod def format_docstring_parameters(params: list[Parameter]) -> str: """Create substitution text for {parameters}""" - add, output = text_accumulator() - for p in params: - if p.docstring: - add(p.render_docstring()) - add('\n') - return output() + return "".join(p.render_docstring() + "\n" for p in params if p.docstring) def format_docstring(self) -> str: assert self.function is not None f = self.function - if f.kind.new_or_init and not f.docstring: - # don't render a docstring at all, no signature, nothing. + # For the following special cases, it does not make sense to render a docstring. + if f.kind in {METHOD_INIT, METHOD_NEW, GETTER, SETTER} and not f.docstring: return f.docstring # Enforce the summary line! @@ -6446,7 +6250,7 @@ def create_cli() -> argparse.ArgumentParser: with writing argument parsing code for builtins and providing introspection signatures ("docstrings") for CPython builtins. -For more information see https://docs.python.org/3/howto/clinic.html""") +For more information see https://devguide.python.org/development-tools/clinic/""") cmdline.add_argument("-f", "--force", action='store_true', help="force output regeneration") cmdline.add_argument("-o", "--output", type=str, diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py new file mode 100644 index 00000000000000..d4e7a0c5cf7b76 --- /dev/null +++ b/Tools/clinic/libclinic/__init__.py @@ -0,0 +1,52 @@ +from typing import Final + +from .errors import ( + ClinicError, +) +from .formatting import ( + SIG_END_MARKER, + c_repr, + docstring_for_c_string, + format_escape, + indent_all_lines, + normalize_snippet, + pprint_words, + suffix_all_lines, + wrap_declarations, + wrapped_c_string_literal, +) + + +__all__ = [ + # Error handling + "ClinicError", + + # Formatting helpers + "SIG_END_MARKER", + "c_repr", + "docstring_for_c_string", + "format_escape", + "indent_all_lines", + "normalize_snippet", + "pprint_words", + "suffix_all_lines", + "wrap_declarations", + "wrapped_c_string_literal", +] + + +CLINIC_PREFIX: Final = "__clinic_" +CLINIC_PREFIXED_ARGS: Final = frozenset( + { + "_keywords", + "_parser", + "args", + "argsbuf", + "fastargs", + "kwargs", + "kwnames", + "nargs", + "noptargs", + "return_value", + } +) diff --git a/Tools/clinic/cpp.py b/Tools/clinic/libclinic/cpp.py similarity index 90% rename from Tools/clinic/cpp.py rename to Tools/clinic/libclinic/cpp.py index 16eee6fc399491..e115d65a88e1b6 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/libclinic/cpp.py @@ -3,6 +3,11 @@ import sys from typing import NoReturn +from .errors import ParseError + + +__all__ = ["Monitor"] + TokenAndCondition = tuple[str, str] TokenStack = list[TokenAndCondition] @@ -32,7 +37,7 @@ class Monitor: Anyway this implementation seems to work well enough for the CPython sources. """ - filename: str | None = None + filename: str _: dc.KW_ONLY verbose: bool = False @@ -59,14 +64,8 @@ def condition(self) -> str: """ return " && ".join(condition for token, condition in self.stack) - def fail(self, *a: object) -> NoReturn: - if self.filename: - filename = " " + self.filename - else: - filename = '' - print("Error at" + filename, "line", self.line_number, ":") - print(" ", ' '.join(str(x) for x in a)) - sys.exit(-1) + def fail(self, msg: str) -> NoReturn: + raise ParseError(msg, filename=self.filename, lineno=self.line_number) def writeline(self, line: str) -> None: self.line_number += 1 @@ -74,7 +73,7 @@ def writeline(self, line: str) -> None: def pop_stack() -> TokenAndCondition: if not self.stack: - self.fail("#" + token + " without matching #if / #ifdef / #ifndef!") + self.fail(f"#{token} without matching #if / #ifdef / #ifndef!") return self.stack.pop() if self.continuation: @@ -145,7 +144,7 @@ def pop_stack() -> TokenAndCondition: if token in {'if', 'ifdef', 'ifndef', 'elif'}: if not condition: - self.fail("Invalid format for #" + token + " line: no argument!") + self.fail(f"Invalid format for #{token} line: no argument!") if token in {'if', 'elif'}: if not is_a_simple_defined(condition): condition = "(" + condition + ")" @@ -155,7 +154,8 @@ def pop_stack() -> TokenAndCondition: else: fields = condition.split() if len(fields) != 1: - self.fail("Invalid format for #" + token + " line: should be exactly one argument!") + self.fail(f"Invalid format for #{token} line: " + "should be exactly one argument!") symbol = fields[0] condition = 'defined(' + symbol + ')' if token == 'ifndef': diff --git a/Tools/clinic/libclinic/errors.py b/Tools/clinic/libclinic/errors.py new file mode 100644 index 00000000000000..afb21b02386fe7 --- /dev/null +++ b/Tools/clinic/libclinic/errors.py @@ -0,0 +1,26 @@ +import dataclasses as dc + + +@dc.dataclass +class ClinicError(Exception): + message: str + _: dc.KW_ONLY + lineno: int | None = None + filename: str | None = None + + def __post_init__(self) -> None: + super().__init__(self.message) + + def report(self, *, warn_only: bool = False) -> str: + msg = "Warning" if warn_only else "Error" + if self.filename is not None: + msg += f" in file {self.filename!r}" + if self.lineno is not None: + msg += f" on line {self.lineno}" + msg += ":\n" + msg += f"{self.message}\n" + return msg + + +class ParseError(ClinicError): + pass diff --git a/Tools/clinic/libclinic/formatting.py b/Tools/clinic/libclinic/formatting.py new file mode 100644 index 00000000000000..8b3ad7ba566bc8 --- /dev/null +++ b/Tools/clinic/libclinic/formatting.py @@ -0,0 +1,173 @@ +"""A collection of string formatting helpers.""" + +import functools +import textwrap +from typing import Final + + +SIG_END_MARKER: Final = "--" + + +def docstring_for_c_string(docstring: str) -> str: + lines = [] + # Turn docstring into a properly quoted C string. + for line in docstring.split("\n"): + lines.append('"') + lines.append(_quoted_for_c_string(line)) + lines.append('\\n"\n') + + if lines[-2] == SIG_END_MARKER: + # If we only have a signature, add the blank line that the + # __text_signature__ getter expects to be there. + lines.append('"\\n"') + else: + lines.pop() + lines.append('"') + return "".join(lines) + + +def _quoted_for_c_string(text: str) -> str: + """Helper for docstring_for_c_string().""" + for old, new in ( + ("\\", "\\\\"), # must be first! + ('"', '\\"'), + ("'", "\\'"), + ): + text = text.replace(old, new) + return text + + +def c_repr(text: str) -> str: + return '"' + text + '"' + + +def wrapped_c_string_literal( + text: str, + *, + width: int = 72, + suffix: str = "", + initial_indent: int = 0, + subsequent_indent: int = 4 +) -> str: + wrapped = textwrap.wrap( + text, + width=width, + replace_whitespace=False, + drop_whitespace=False, + break_on_hyphens=False, + ) + separator = c_repr(suffix + "\n" + subsequent_indent * " ") + return initial_indent * " " + c_repr(separator.join(wrapped)) + + +def _add_prefix_and_suffix(text: str, *, prefix: str = "", suffix: str = "") -> str: + """Return 'text' with 'prefix' prepended and 'suffix' appended to all lines. + + If the last line is empty, it remains unchanged. + If text is blank, return text unchanged. + + (textwrap.indent only adds to non-blank lines.) + """ + *split, last = text.split("\n") + lines = [prefix + line + suffix + "\n" for line in split] + if last: + lines.append(prefix + last + suffix) + return "".join(lines) + + +def indent_all_lines(text: str, prefix: str) -> str: + return _add_prefix_and_suffix(text, prefix=prefix) + + +def suffix_all_lines(text: str, suffix: str) -> str: + return _add_prefix_and_suffix(text, suffix=suffix) + + +def pprint_words(items: list[str]) -> str: + if len(items) <= 2: + return " and ".join(items) + return ", ".join(items[:-1]) + " and " + items[-1] + + +def _strip_leading_and_trailing_blank_lines(text: str) -> str: + lines = text.rstrip().split("\n") + while lines: + line = lines[0] + if line.strip(): + break + del lines[0] + return "\n".join(lines) + + +@functools.lru_cache() +def normalize_snippet(text: str, *, indent: int = 0) -> str: + """ + Reformats 'text': + * removes leading and trailing blank lines + * ensures that it does not end with a newline + * dedents so the first nonwhite character on any line is at column "indent" + """ + text = _strip_leading_and_trailing_blank_lines(text) + text = textwrap.dedent(text) + if indent: + text = textwrap.indent(text, " " * indent) + return text + + +def format_escape(text: str) -> str: + # double up curly-braces, this string will be used + # as part of a format_map() template later + text = text.replace("{", "{{") + text = text.replace("}", "}}") + return text + + +def wrap_declarations(text: str, length: int = 78) -> str: + """ + A simple-minded text wrapper for C function declarations. + + It views a declaration line as looking like this: + xxxxxxxx(xxxxxxxxx,xxxxxxxxx) + If called with length=30, it would wrap that line into + xxxxxxxx(xxxxxxxxx, + xxxxxxxxx) + (If the declaration has zero or one parameters, this + function won't wrap it.) + + If this doesn't work properly, it's probably better to + start from scratch with a more sophisticated algorithm, + rather than try and improve/debug this dumb little function. + """ + lines = [] + for line in text.split("\n"): + prefix, _, after_l_paren = line.partition("(") + if not after_l_paren: + lines.append(line) + continue + in_paren, _, after_r_paren = after_l_paren.partition(")") + if not _: + lines.append(line) + continue + if "," not in in_paren: + lines.append(line) + continue + parameters = [x.strip() + ", " for x in in_paren.split(",")] + prefix += "(" + if len(prefix) < length: + spaces = " " * len(prefix) + else: + spaces = " " * 4 + + while parameters: + line = prefix + first = True + while parameters: + if not first and (len(line) + len(parameters[0]) > length): + break + line += parameters.pop(0) + first = False + if not parameters: + line = line.rstrip(", ") + ")" + after_r_paren + lines.append(line.rstrip()) + prefix = spaces + return "\n".join(lines) diff --git a/Tools/freeze/README b/Tools/freeze/README index 9b3ea1f2c723b1..516077bc7daa89 100644 --- a/Tools/freeze/README +++ b/Tools/freeze/README @@ -218,6 +218,11 @@ source tree). It is possible to create frozen programs that don't have a console window, by specifying the option '-s windows'. See the Usage below. +Usage under macOS +----------------- + +On macOS the freeze tool is not supported for framework builds. + Usage ----- diff --git a/Tools/freeze/freeze.py b/Tools/freeze/freeze.py index bc5e43f4853deb..de9772732cdb5d 100755 --- a/Tools/freeze/freeze.py +++ b/Tools/freeze/freeze.py @@ -136,6 +136,11 @@ def main(): makefile = 'Makefile' subsystem = 'console' + if sys.platform == "darwin" and sysconfig.get_config_var("PYTHONFRAMEWORK"): + print(f"{sys.argv[0]} cannot be used with framework builds of Python", file=sys.stderr) + sys.exit(1) + + # parse command line by first replacing any "-i" options with the # file contents. pos = 1 diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 21f9c848cc6be5..4357dc86d9d356 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -3,7 +3,7 @@ - + diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 30bfb31471c7b2..00295c984d1bb6 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -139,10 +139,15 @@ def compile_c_extension( ] include_dirs = [ str(MOD_DIR.parent.parent.parent / "Include" / "internal"), + str(MOD_DIR.parent.parent.parent / "Include" / "internal" / "mimalloc"), str(MOD_DIR.parent.parent.parent / "Parser"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"), ] + if sys.platform == "win32": + # HACK: The location of pyconfig.h has moved within our build, and + # setuptools hasn't updated for it yet. So add the path manually for now + include_dirs.append(pathlib.Path(sysconfig.get_config_h_filename()).parent) extension = Extension( extension_name, sources=[generated_source_path], diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 591baac33c7e8f..b89f86a35d6115 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.7.0 +mypy==1.8.0 # needed for peg_generator: types-psutil==5.9.5.17 -types-setuptools==68.2.0.0 +types-setuptools==69.0.0.0 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 0fcc72ac69073d..0e6e16ae198162 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.88.1 +hypothesis==6.92.2 diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 360b7c720bd1f0..80a1280c025aca 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -386,6 +386,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: trace_too_short = self._data["Optimization trace too short"] inner_loop = self._data["Optimization inner loop"] recursive_call = self._data["Optimization recursive call"] + low_confidence = self._data["Optimization low confidence"] return { "Optimization attempts": (attempts, None), @@ -396,6 +397,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: "Trace too short": (trace_too_short, attempts), "Inner loop found": (inner_loop, attempts), "Recursive call": (recursive_call, attempts), + "Low confidence": (low_confidence, attempts), "Traces executed": (executed, None), "Uops executed": (uops, executed), } diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 8ef63c6dcd9ddc..beb857f69e40da 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -298,102 +298,68 @@ AddType application/wasm wasm ## WASI (wasm32-wasi) -WASI builds require the [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+. -See `.devcontainer/Dockerfile` for an example of how to download and -install the WASI SDK. +**NOTE**: The instructions below assume a Unix-based OS due to cross-compilation for CPython being set up for `./configure`. -### Build +### Prerequisites + +Developing for WASI requires two additional tools to be installed beyond the typical tools required to build CPython: + +1. The [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+ +2. A WASI host/runtime ([wasmtime](https://wasmtime.dev) 14+ is recommended and what the instructions below assume) -The script ``wasi-env`` sets necessary compiler and linker flags as well as -``pkg-config`` overrides. The script assumes that WASI-SDK is installed in -``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``. +All of this is provided in the [devcontainer](https://devguide.python.org/getting-started/setup-building/#contribute-using-github-codespaces) if you don't want to install these tools locally. -There are two scripts you can use to do a WASI build from a source checkout. You can either use: +### Building +Building for WASI requires doing a cross-build where you have a "build" Python to help produce a WASI build of CPython (technically it's a "host x host" cross-build because the build Python is also the target Python while the host build is the WASI build; yes, it's confusing terminology). In the end you should have a build Python in `cross-build/build` and a WASI build in `cross-build/wasm32-wasi`. + +The easiest way to do a build is to use the `wasi.py` script. You can either have it perform the entire build process from start to finish in one step, or you can do it in discrete steps that mirror running `configure` and `make` for each of the two builds of Python you end up producing (which are beneficial when you only need to do a specific step after getting a complete build, e.g. editing some code and you just need to run `make` for the WASI build). The script is designed to self-document what actions it is performing on your behalf, both as a way to check its work but also for educaitonal purposes. + +The discrete steps for building via `wasi.py` are: ```shell -./Tools/wasm/wasm_build.py wasi build +python Tools/wasm/wasi.py configure-build-python +python Tools/wasm/wasi.py make-build-python +python Tools/wasm/wasi.py configure-host +python Tools/wasm/wasi.py make-host ``` -or: +To do it all in a single command, run: ```shell -./Tools/wasm/build_wasi.sh +python Tools/wasm/wasi.py build ``` -The commands are equivalent to the following steps: - -- Make sure `Modules/Setup.local` exists -- Make sure the necessary build tools are installed: - - [WASI SDK](https://github.com/WebAssembly/wasi-sdk) (which ships with `clang`) - - `make` - - `pkg-config` (on Linux) -- Create the build Python - - `mkdir -p builddir/build` - - `pushd builddir/build` - - Get the build platform - - Python: `sysconfig.get_config_var("BUILD_GNU_TYPE")` - - Shell: `../../config.guess` - - `../../configure -C` - - `make all` - - ```PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` ``` - - `popd` -- Create the host/WASI Python - - `mkdir builddir/wasi` - - `pushd builddir/wasi` - - `../../Tools/wasm/wasi-env ../../configure -C --host=wasm32-unknown-wasi --build=$(../../config.guess) --with-build-python=../build/python` - - `CONFIG_SITE=../../Tools/wasm/config.site-wasm32-wasi` - - `HOSTRUNNER="wasmtime run --mapdir /::$(dirname $(dirname $(pwd))) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/python.wasm --"` - - Maps the source checkout to `/` in the WASI runtime - - Stdlib gets loaded from `/Lib` - - Gets `_sysconfigdata__wasi_wasm32-wasi.py` on to `sys.path` via `PYTHONPATH` - - Set by `wasi-env` - - `WASI_SDK_PATH` - - `WASI_SYSROOT` - - `CC` - - `CPP` - - `CXX` - - `LDSHARED` - - `AR` - - `RANLIB` - - `CFLAGS` - - `LDFLAGS` - - `PKG_CONFIG_PATH` - - `PKG_CONFIG_LIBDIR` - - `PKG_CONFIG_SYSROOT_DIR` - - `PATH` - - `make all` - +That will: -### Running - -If you followed the instructions above, you can run the interpreter via e.g., `wasmtime` from within the `Tools/wasi` directory (make sure to set/change `$PYTHON_VERSION` and do note the paths are relative to running in`builddir/wasi` for simplicity only): +1. Run `configure` for the build Python (same as `wasi.py configure-build-python`) +2. Run `make` for the build Python (`wasi.py make-build-python`) +3. Run `configure` for the WASI build (`wasi.py configure-host`) +4. Run `make` for the WASI build (`wasi.py make-host`) +See the `--help` for the various options available for each of the subcommands which controls things like the location of the WASI SDK, the command to use with the WASI host/runtime, etc. Also note that you can use `--` as a separator for any of the `configure`-related commands -- including `build` itself -- to pass arguments to the underlying `configure` call. For example, if you want a pydebug build that also caches the results from `configure`, you can do: ```shell -wasmtime run --mapdir /::../.. --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION python.wasm -- +python Tools/wasm/wasi.py build -- -C --with-pydebug ``` -There are also helpers provided by `Tools/wasm/wasm_build.py` as listed below. Also, if you used `Tools/wasm/build_wasi.sh`, a `run_wasi.sh` file will be created in `builddir/wasi` which will run the above command for you (it also uses absolute paths, so it can be executed from anywhere). - -#### REPL - +The `wasi.py` script is able to infer details from the build Python, and so you only technically need to specify `--with-pydebug` once via `configure-build-python` as this will lead to `configure-host` detecting its use if you use the discrete steps: ```shell -./Tools/wasm/wasm_build.py wasi repl +python Tools/wasm/wasi.py configure-build-python -- -C --with-pydebug +python Tools/wasm/wasi.py make-build-python +python Tools/wasm/wasi.py configure-host -- -C +python Tools/wasm/wasi.py make-host ``` -#### Tests +### Running + +If you used `wasi.py` to do your build then there will be a `cross-build/wasm32-wasi/python.sh` file which you can use to run the `python.wasm` file (see the output from the `configure-host` subcommand): ```shell -./Tools/wasm/wasm_build.py wasi test +cross-build/wasm32-wasi/python.sh --version ``` -### Debugging +While you _can_ run `python.wasm` directly, Python will fail to start up without certain things being set (e.g. `PYTHONPATH` for `sysconfig` data). As such, the `python.sh` file records these details for you. -* ``wasmtime run -g`` generates debugging symbols for gdb and lldb. The - feature is currently broken, see - https://github.com/bytecodealliance/wasmtime/issues/4669 . -* The environment variable ``RUST_LOG=wasi_common`` enables debug and - trace logging. -## Detect WebAssembly builds +## Detecting WebAssembly builds ### Python code @@ -402,15 +368,17 @@ import os, sys if sys.platform == "emscripten": # Python on Emscripten + ... if sys.platform == "wasi": # Python on WASI + ... if os.name == "posix": # WASM platforms identify as POSIX-like. # Windows does not provide os.uname(). machine = os.uname().machine if machine.startswith("wasm"): - # WebAssembly (wasm32, wasm64 in the future) + # WebAssembly (wasm32, wasm64 potentially in the future) ``` ```python diff --git a/Tools/wasm/mypy.ini b/Tools/wasm/mypy.ini index c62598f89eba69..4de0a30c260f5f 100644 --- a/Tools/wasm/mypy.ini +++ b/Tools/wasm/mypy.ini @@ -1,5 +1,5 @@ [mypy] -files = Tools/wasm +files = Tools/wasm/wasm_*.py pretty = True show_traceback = True @@ -9,6 +9,3 @@ python_version = 3.8 # Be strict... strict = True enable_error_code = truthy-bool,ignore-without-code - -# except for incomplete defs, which are useful for module authors: -disallow_incomplete_defs = False diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py new file mode 100644 index 00000000000000..34c0e9375e24c8 --- /dev/null +++ b/Tools/wasm/wasi.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count +import pathlib +import shutil +import subprocess +import sys +import sysconfig +import tempfile + + +CHECKOUT = pathlib.Path(__file__).parent.parent.parent +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +BUILD_DIR = CROSS_BUILD_DIR / "build" +HOST_TRIPLE = "wasm32-wasi" +HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + print("🌎 Environment changes:") + for key in sorted(env_diff.keys()): + print(f" {key}={env_diff[key]}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + def decorator(func): + @functools.wraps(func) + def wrapper(context): + try: + tput_output = subprocess.check_output(["tput", "cols"], + encoding="utf-8") + terminal_width = int(tput_output.strip()) + except subprocess.CalledProcessError: + terminal_width = 80 + print("⎯" * terminal_width) + print("📁", working_dir) + if clean_ok and context.clean and working_dir.exists(): + print(f"🚮 Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, quiet, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + print("❯", " ".join(map(str, command))) + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", + delete=False, + prefix="cpython-wasi-", + suffix=".log") + stderr = subprocess.STDOUT + print(f"📝 Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_platform(): + """The name of the build/host platform.""" + # Can also be found via `config.guess`.` + return sysconfig.get_config_var("BUILD_GNU_TYPE") + + +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " + f"{BUILD_DIR}") + + return binary + + +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + local_setup = CHECKOUT / "Modules" / "Setup.local" + if local_setup.exists(): + print(f"👍 {local_setup} exists ...") + else: + print(f"📝 Touching {local_setup} ...") + local_setup.touch() + + configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, quiet=context.quiet) + + +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], + quiet=context.quiet) + + binary = build_python_path() + cmd = [binary, "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + print(f"🎉 {binary} {version}") + + +def find_wasi_sdk(): + """Find the path to wasi-sdk.""" + if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): + return pathlib.Path(wasi_sdk_path) + elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists(): + return default_path + + +def wasi_sdk_env(context): + """Calculate environment variables for building with wasi-sdk.""" + wasi_sdk_path = context.wasi_sdk_path + sysroot = wasi_sdk_path / "share" / "wasi-sysroot" + env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", + "LDSHARED": "wasm-ld", "AR": "llvm-ar", "RANLIB": "ranlib"} + + for env_var, binary_name in list(env.items()): + env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) + + if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): + for compiler in ["CC", "CPP", "CXX"]: + env[compiler] += f" --sysroot={sysroot}" + + env["PKG_CONFIG_PATH"] = "" + env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( + map(os.fsdecode, + [sysroot / "lib" / "pkgconfig", + sysroot / "share" / "pkgconfig"])) + env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) + + env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) + env["WASI_SYSROOT"] = os.fsdecode(sysroot) + + env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"]]) + + return env + + +@subdir(HOST_DIR, clean_ok=True) +def configure_wasi_python(context, working_dir): + """Configure the WASI/host build.""" + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): + raise ValueError("WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk") + + config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") + + wasi_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" + lib_dir = os.fsdecode(lib_dirs[0]) + pydebug = lib_dir.endswith("-pydebug") + python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + if pydebug: + sysconfig_data += "-pydebug" + + # Use PYTHONPATH to include sysconfig data which must be anchored to the + # WASI guest's `/` directory. + host_runner = context.host_runner.format(GUEST_DIR="/", + HOST_DIR=CHECKOUT, + ENV_VAR_NAME="PYTHONPATH", + ENV_VAR_VALUE=f"/{sysconfig_data}", + PYTHON_WASM=working_dir / "python.wasm") + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. + configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), + f"--host={HOST_TRIPLE}", + f"--build={build_platform()}", + f"--with-build-python={build_python}"] + if pydebug: + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call(configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + quiet=context.quiet) + + exec_script = working_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} "$@"\n') + exec_script.chmod(0o755) + print(f"🏃‍♀️ Created {exec_script} ... ") + sys.stdout.flush() + + +@subdir(HOST_DIR) +def make_wasi_python(context, working_dir): + """Run `make` for the WASI/host build.""" + call(["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + quiet=context.quiet) + + exec_script = working_dir / "python.sh" + subprocess.check_call([exec_script, "--version"]) + + +def build_all(context): + """Build everything.""" + steps = [configure_build_python, make_build_python, configure_wasi_python, + make_wasi_python] + for step in steps: + step(context) + + +def main(): + default_host_runner = (f"{shutil.which('wasmtime')} run " + # Make sure the stack size will work for a pydebug + # build. + # The 8388608 value comes from `ulimit -s` under Linux + # which equates to 8291 KiB. + "--wasm max-wasm-stack=8388608 " + # Enable thread support. + "--wasm threads=y --wasi threads=y " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE} " + # Path to the WASM binary. + "{PYTHON_WASM}") + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser("configure-build-python", + help="Run `configure` for the " + "build Python") + make_build = subcommands.add_parser("make-build-python", + help="Run `make` for the build Python") + configure_host = subcommands.add_parser("configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)") + make_host = subcommands.add_parser("make-host", + help="Run `make` for the host/WASI") + for subcommand in build, configure_build, make_build, configure_host, make_host: + subcommand.add_argument("--quiet", action="store_true", default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("--clean", action="store_true", default=False, + dest="clean", + help="Delete any relevant directories before building") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + for subcommand in build, configure_host: + subcommand.add_argument("--wasi-sdk", type=pathlib.Path, + dest="wasi_sdk_path", + default=find_wasi_sdk(), + help="Path to wasi-sdk; defaults to " + "$WASI_SDK_PATH or /opt/wasi-sdk") + subcommand.add_argument("--host-runner", action="store", + default=default_host_runner, dest="host_runner", + help="Command template for running the WebAssembly " + "code (default meant for wasmtime 14 or newer: " + f"`{default_host_runner}`)") + + context = parser.parse_args() + + dispatch = {"configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build": build_all} + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/configure b/configure index 319009537f461c..3cc9aecafad13e 100755 --- a/configure +++ b/configure @@ -769,6 +769,8 @@ MODULE__MULTIPROCESSING_FALSE MODULE__MULTIPROCESSING_TRUE MODULE__ZONEINFO_FALSE MODULE__ZONEINFO_TRUE +MODULE__XXINTERPQUEUES_FALSE +MODULE__XXINTERPQUEUES_TRUE MODULE__XXINTERPCHANNELS_FALSE MODULE__XXINTERPCHANNELS_TRUE MODULE__XXSUBINTERPRETERS_FALSE @@ -1080,6 +1082,7 @@ with_dsymutil with_address_sanitizer with_memory_sanitizer with_undefined_behavior_sanitizer +with_thread_sanitizer with_hash_algorithm with_tzpath with_libs @@ -1858,6 +1861,8 @@ Optional Packages: --with-undefined-behavior-sanitizer enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no) + --with-thread-sanitizer enable ThreadSanitizer data race detector, 'tsan' + (default is no) --with-hash-algorithm=[fnv|siphash13|siphash24] select hash algorithm for use in Python/pyhash.c (default is SipHash13) @@ -12504,6 +12509,28 @@ with_ubsan="no" fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-thread-sanitizer" >&5 +printf %s "checking for --with-thread-sanitizer... " >&6; } + +# Check whether --with-thread_sanitizer was given. +if test ${with_thread_sanitizer+y} +then : + withval=$with_thread_sanitizer; +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +printf "%s\n" "$withval" >&6; } +BASECFLAGS="-fsanitize=thread $BASECFLAGS" +LDFLAGS="-fsanitize=thread $LDFLAGS" +with_tsan="yes" + +else $as_nop + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +with_tsan="no" + +fi + + # Set info about shared libraries. @@ -16891,6 +16918,8 @@ printf "%s\n" "#define WITH_MIMALLOC 1" >>confdefs.h MIMALLOC_HEADERS='$(MIMALLOC_HEADERS)' +elif test "$disable_gil" = "yes"; then + as_fn_error $? "--disable-gil requires mimalloc memory allocator (--with-mimalloc)." "$LINENO" 5 fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_mimalloc" >&5 @@ -17223,6 +17252,12 @@ if test "x$ac_cv_func_clock" = xyes then : printf "%s\n" "#define HAVE_CLOCK 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "closefrom" "ac_cv_func_closefrom" +if test "x$ac_cv_func_closefrom" = xyes +then : + printf "%s\n" "#define HAVE_CLOSEFROM 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "close_range" "ac_cv_func_close_range" if test "x$ac_cv_func_close_range" = xyes @@ -17769,6 +17804,12 @@ if test "x$ac_cv_func_posix_spawnp" = xyes then : printf "%s\n" "#define HAVE_POSIX_SPAWNP 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "posix_spawn_file_actions_addclosefrom_np" "ac_cv_func_posix_spawn_file_actions_addclosefrom_np" +if test "x$ac_cv_func_posix_spawn_file_actions_addclosefrom_np" = xyes +then : + printf "%s\n" "#define HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pread" "ac_cv_func_pread" if test "x$ac_cv_func_pread" = xyes @@ -21789,6 +21830,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #if defined(MAJOR_IN_MKDEV) #include #elif defined(MAJOR_IN_SYSMACROS) +#include #include #else #include @@ -23971,7 +24013,6 @@ fi - # Check whether --with-readline was given. if test ${with_readline+y} then : @@ -23994,22 +24035,6 @@ else $as_nop fi -# gh-105323: Need to handle the macOS editline as an alias of readline. -case $ac_sys_system/$ac_sys_release in #( - Darwin/*) : - ac_fn_c_check_type "$LINENO" "Function" "ac_cv_type_Function" "#include -" -if test "x$ac_cv_type_Function" = xyes -then : - printf "%s\n" "#define WITH_APPLE_EDITLINE 1" >>confdefs.h - -fi - ;; #( - *) : - - ;; -esac - if test "x$with_readline" = xreadline then : @@ -27479,7 +27504,12 @@ then : else $as_nop - rpath_arg="-Wl,-rpath=" + if test "$ac_sys_system" = "Darwin" + then + rpath_arg="-Wl,-rpath," + else + rpath_arg="-Wl,-rpath=" + fi fi @@ -27902,6 +27932,9 @@ printf "%s\n" "$TEST_MODULES" >&6; } # libatomic __atomic_fetch_or_8(), or not, depending on the C compiler and the # compiler flags. # +# gh-112779: On RISC-V, GCC 12 and earlier require libatomic support for 1-byte +# and 2-byte operations, but not for 8-byte operations. +# # Avoid #include or #include . The header # requires header which is only written below by AC_OUTPUT below. # If the check is done after AC_OUTPUT, modifying LIBS has no effect @@ -27941,12 +27974,19 @@ typedef intptr_t Py_ssize_t; int main() { - uint64_t byte; - _Py_atomic_store_uint64(&byte, 2); - if (_Py_atomic_or_uint64(&byte, 8) != 2) { + uint64_t value; + _Py_atomic_store_uint64(&value, 2); + if (_Py_atomic_or_uint64(&value, 8) != 2) { + return 1; // error + } + if (_Py_atomic_load_uint64(&value) != 10) { return 1; // error } - if (_Py_atomic_load_uint64(&byte) != 10) { + uint8_t byte = 0xb8; + if (_Py_atomic_or_uint8(&byte, 0x2d) != 0xb8) { + return 1; // error + } + if (_Py_atomic_load_uint8(&byte) != 0xbd) { return 1; // error } return 0; // all good @@ -28024,6 +28064,7 @@ case $ac_sys_system in #( py_cv_module__tkinter=n/a py_cv_module__xxsubinterpreters=n/a py_cv_module__xxinterpchannels=n/a + py_cv_module__xxinterpqueues=n/a py_cv_module_grp=n/a py_cv_module_pwd=n/a py_cv_module_resource=n/a @@ -28523,6 +28564,28 @@ then : +fi + + + if test "$py_cv_module__xxinterpqueues" != "n/a" +then : + py_cv_module__xxinterpqueues=yes +fi + if test "$py_cv_module__xxinterpqueues" = yes; then + MODULE__XXINTERPQUEUES_TRUE= + MODULE__XXINTERPQUEUES_FALSE='#' +else + MODULE__XXINTERPQUEUES_TRUE='#' + MODULE__XXINTERPQUEUES_FALSE= +fi + + as_fn_append MODULE_BLOCK "MODULE__XXINTERPQUEUES_STATE=$py_cv_module__xxinterpqueues$as_nl" + if test "x$py_cv_module__xxinterpqueues" = xyes +then : + + + + fi @@ -30759,6 +30822,10 @@ if test -z "${MODULE__XXINTERPCHANNELS_TRUE}" && test -z "${MODULE__XXINTERPCHAN as_fn_error $? "conditional \"MODULE__XXINTERPCHANNELS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__XXINTERPQUEUES_TRUE}" && test -z "${MODULE__XXINTERPQUEUES_FALSE}"; then + as_fn_error $? "conditional \"MODULE__XXINTERPQUEUES\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__ZONEINFO_TRUE}" && test -z "${MODULE__ZONEINFO_FALSE}"; then as_fn_error $? "conditional \"MODULE__ZONEINFO\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index b78472e04846b7..6a80a5d29a04ef 100644 --- a/configure.ac +++ b/configure.ac @@ -3067,6 +3067,24 @@ AC_MSG_RESULT([no]) with_ubsan="no" ]) +AC_MSG_CHECKING([for --with-thread-sanitizer]) +AC_ARG_WITH( + [thread_sanitizer], + [AS_HELP_STRING( + [--with-thread-sanitizer], + [enable ThreadSanitizer data race detector, 'tsan' (default is no)] + )], +[ +AC_MSG_RESULT([$withval]) +BASECFLAGS="-fsanitize=thread $BASECFLAGS" +LDFLAGS="-fsanitize=thread $LDFLAGS" +with_tsan="yes" +], +[ +AC_MSG_RESULT([no]) +with_tsan="no" +]) + # Set info about shared libraries. AC_SUBST([SHLIB_SUFFIX]) AC_SUBST([LDSHARED]) @@ -4558,6 +4576,8 @@ if test "$with_mimalloc" != no; then with_mimalloc=yes AC_DEFINE([WITH_MIMALLOC], [1], [Define if you want to compile in mimalloc memory allocator.]) AC_SUBST([MIMALLOC_HEADERS], ['$(MIMALLOC_HEADERS)']) +elif test "$disable_gil" = "yes"; then + AC_MSG_ERROR([--disable-gil requires mimalloc memory allocator (--with-mimalloc).]) fi AC_MSG_RESULT([$with_mimalloc]) @@ -4743,7 +4763,7 @@ fi # checks for library functions AC_CHECK_FUNCS([ \ - accept4 alarm bind_textdomain_codeset chmod chown clock close_range confstr \ + accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ @@ -4755,6 +4775,7 @@ AC_CHECK_FUNCS([ \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ pipe2 plock poll posix_fadvise posix_fallocate posix_spawn posix_spawnp \ + posix_spawn_file_actions_addclosefrom_np \ pread preadv preadv2 pthread_condattr_setclock pthread_init pthread_kill \ pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ @@ -5099,6 +5120,7 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #if defined(MAJOR_IN_MKDEV) #include #elif defined(MAJOR_IN_SYSMACROS) +#include #include #else #include @@ -5914,7 +5936,6 @@ dnl library (tinfo ncursesw ncurses termcap). We now assume that libreadline dnl or readline.pc provide correct linker information. AH_TEMPLATE([WITH_EDITLINE], [Define to build the readline module against libedit.]) -AH_TEMPLATE([WITH_APPLE_EDITLINE], [Define to build the readline module against Apple BSD editline.]) AC_ARG_WITH( [readline], @@ -5931,15 +5952,6 @@ AC_ARG_WITH( [with_readline=readline] ) -# gh-105323: Need to handle the macOS editline as an alias of readline. -AS_CASE([$ac_sys_system/$ac_sys_release], - [Darwin/*], [AC_CHECK_TYPE([Function], - [AC_DEFINE([WITH_APPLE_EDITLINE])], - [], - [@%:@include ])], - [] -) - AS_VAR_IF([with_readline], [readline], [ PKG_CHECK_MODULES([LIBREADLINE], [readline], [ LIBREADLINE=readline @@ -6815,7 +6827,12 @@ AX_CHECK_OPENSSL([have_openssl=yes],[have_openssl=no]) AS_VAR_IF([GNULD], [yes], [ rpath_arg="-Wl,--enable-new-dtags,-rpath=" ], [ - rpath_arg="-Wl,-rpath=" + if test "$ac_sys_system" = "Darwin" + then + rpath_arg="-Wl,-rpath," + else + rpath_arg="-Wl,-rpath=" + fi ]) AC_MSG_CHECKING([for --with-openssl-rpath]) @@ -7033,6 +7050,9 @@ AC_SUBST([TEST_MODULES]) # libatomic __atomic_fetch_or_8(), or not, depending on the C compiler and the # compiler flags. # +# gh-112779: On RISC-V, GCC 12 and earlier require libatomic support for 1-byte +# and 2-byte operations, but not for 8-byte operations. +# # Avoid #include or #include . The header # requires header which is only written below by AC_OUTPUT below. # If the check is done after AC_OUTPUT, modifying LIBS has no effect @@ -7062,12 +7082,19 @@ typedef intptr_t Py_ssize_t; int main() { - uint64_t byte; - _Py_atomic_store_uint64(&byte, 2); - if (_Py_atomic_or_uint64(&byte, 8) != 2) { + uint64_t value; + _Py_atomic_store_uint64(&value, 2); + if (_Py_atomic_or_uint64(&value, 8) != 2) { + return 1; // error + } + if (_Py_atomic_load_uint64(&value) != 10) { + return 1; // error + } + uint8_t byte = 0xb8; + if (_Py_atomic_or_uint8(&byte, 0x2d) != 0xb8) { return 1; // error } - if (_Py_atomic_load_uint64(&byte) != 10) { + if (_Py_atomic_load_uint8(&byte) != 0xbd) { return 1; // error } return 0; // all good @@ -7118,6 +7145,7 @@ AS_CASE([$ac_sys_system], [_tkinter], [_xxsubinterpreters], [_xxinterpchannels], + [_xxinterpqueues], [grp], [pwd], [resource], @@ -7234,6 +7262,7 @@ PY_STDLIB_MOD_SIMPLE([_struct]) PY_STDLIB_MOD_SIMPLE([_typing]) PY_STDLIB_MOD_SIMPLE([_xxsubinterpreters]) PY_STDLIB_MOD_SIMPLE([_xxinterpchannels]) +PY_STDLIB_MOD_SIMPLE([_xxinterpqueues]) PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules diff --git a/pyconfig.h.in b/pyconfig.h.in index bf708926e22c43..d8a9f68951afbd 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -157,6 +157,9 @@ /* Define to 1 if you have the `clock_settime' function. */ #undef HAVE_CLOCK_SETTIME +/* Define to 1 if you have the `closefrom' function. */ +#undef HAVE_CLOSEFROM + /* Define to 1 if you have the `close_range' function. */ #undef HAVE_CLOSE_RANGE @@ -902,6 +905,10 @@ /* Define to 1 if you have the `posix_spawnp' function. */ #undef HAVE_POSIX_SPAWNP +/* Define to 1 if you have the `posix_spawn_file_actions_addclosefrom_np' + function. */ +#undef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP + /* Define to 1 if you have the `pread' function. */ #undef HAVE_PREAD @@ -1800,9 +1807,6 @@ /* Define if WINDOW in curses.h offers a field _flags. */ #undef WINDOW_HAS_FLAGS -/* Define to build the readline module against Apple BSD editline. */ -#undef WITH_APPLE_EDITLINE - /* Define if you want build the _decimal module using a coroutine-local rather than a thread-local context */ #undef WITH_DECIMAL_CONTEXTVAR