From 6702d2bf6edcd5b5415e17837383623b9d76a5b8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 28 Mar 2024 17:40:58 +0100 Subject: [PATCH 001/143] gh-114331: Skip decimal test_maxcontext_exact_arith on s390x (#117326) --- Lib/test/test_decimal.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index f23ea8af0c8772..05dcb25a7e5950 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -38,7 +38,8 @@ check_disallow_instantiation) from test.support import (TestFailed, run_with_locale, cpython_only, - darwin_malloc_err_warning, is_emscripten) + darwin_malloc_err_warning, is_emscripten, + skip_on_s390x) from test.support.import_helper import import_fresh_module from test.support import threading_helper from test.support import warnings_helper @@ -5650,6 +5651,9 @@ def __abs__(self): @unittest.skipIf(check_sanitizer(address=True, memory=True), "ASAN/MSAN sanitizer defaults to crashing " "instead of returning NULL for malloc failure.") + # gh-114331: The test allocates 784 271 641 GiB and mimalloc does not fail + # to allocate it when using mimalloc on s390x. + @skip_on_s390x def test_maxcontext_exact_arith(self): # Make sure that exact operations do not raise MemoryError due From efcc96844e7c66fcd6c23ac2d557ca141614ce9a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 28 Mar 2024 11:23:29 -0700 Subject: [PATCH 002/143] gh-69201: Separate stdout and stderr stream in test_pdb (#117308) --- Lib/test/test_pdb.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index eedbcec1e66dcb..2d057e2647f13c 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2603,12 +2603,12 @@ def _run_pdb(self, pdb_args, commands, cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**env, 'PYTHONIOENCODING': 'utf-8'} ) as proc: stdout, stderr = proc.communicate(str.encode(commands)) - stdout = stdout and bytes.decode(stdout) - stderr = stderr and bytes.decode(stderr) + stdout = bytes.decode(stdout) if isinstance(stdout, bytes) else stdout + stderr = bytes.decode(stderr) if isinstance(stderr, bytes) else stderr self.assertEqual( proc.returncode, expected_returncode, @@ -2756,7 +2756,7 @@ def test_issue7964(self): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, ) self.addCleanup(proc.stdout.close) stdout, stderr = proc.communicate(b'quit\n') @@ -2840,7 +2840,7 @@ def start_pdb(): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env={**os.environ, 'PYTHONIOENCODING': 'utf-8'} ) self.addCleanup(proc.stdout.close) @@ -2870,7 +2870,7 @@ def start_pdb(): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'} ) self.addCleanup(proc.stdout.close) @@ -2886,10 +2886,10 @@ def test_issue16180(self): stdout, stderr = self.run_pdb_script( script, commands ) - self.assertIn(expected, stdout, + self.assertIn(expected, stderr, '\n\nExpected:\n{}\nGot:\n{}\n' 'Fail to handle a syntax error in the debuggee.' - .format(expected, stdout)) + .format(expected, stderr)) def test_issue84583(self): # A syntax error from ast.literal_eval should not make pdb exit. @@ -2900,11 +2900,12 @@ def test_issue84583(self): quit """ stdout, stderr = self.run_pdb_script(script, commands) - # The code should appear 3 times in the stdout: - # 1. when pdb starts - # 2. when the exception is raised, in trackback - # 3. in where command - self.assertEqual(stdout.count("ast.literal_eval('')"), 3) + # The code should appear 3 times in the stdout/stderr: + # 1. when pdb starts (stdout) + # 2. when the exception is raised, in trackback (stderr) + # 3. in where command (stdout) + self.assertEqual(stdout.count("ast.literal_eval('')"), 2) + self.assertEqual(stderr.count("ast.literal_eval('')"), 1) def test_issue26053(self): # run command of pdb prompt echoes the correct args @@ -3133,9 +3134,9 @@ def test_dir_as_script(self): def test_invalid_cmd_line_options(self): stdout, stderr = self._run_pdb(["-c"], "", expected_returncode=2) - self.assertIn(f"pdb: error: argument -c/--command: expected one argument", stdout.split('\n')[1]) + self.assertIn(f"pdb: error: argument -c/--command: expected one argument", stderr.split('\n')[1]) stdout, stderr = self._run_pdb(["--spam", "-m", "pdb"], "", expected_returncode=2) - self.assertIn(f"pdb: error: unrecognized arguments: --spam", stdout.split('\n')[1]) + self.assertIn(f"pdb: error: unrecognized arguments: --spam", stderr.split('\n')[1]) def test_blocks_at_first_code_line(self): script = """ @@ -3190,7 +3191,7 @@ def test_file_modified_after_execution_with_multiple_instances(self): cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}, ) as proc: stdout, _ = proc.communicate(str.encode(commands)) From 29829b58a8328a7c2ccacaa74c1d7d120a5e5ca5 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 28 Mar 2024 19:59:12 +0000 Subject: [PATCH 003/143] gh-117294: Report DocTestCase as skipped if all examples in the doctest are skipped (GH-117297) --- Doc/library/doctest.rst | 6 ++- Lib/doctest.py | 7 +-- Lib/test/test_doctest/sample_doctest_skip.py | 49 +++++++++++++++++++ Lib/test/test_doctest/test_doctest.py | 22 +++++++++ Lib/test/test_doctest/test_doctest_skip.txt | 4 ++ Lib/test/test_zipimport_support.py | 4 +- ...-03-27-16-43-42.gh-issue-117294.wbXNFv.rst | 2 + 7 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 Lib/test/test_doctest/sample_doctest_skip.py create mode 100644 Lib/test/test_doctest/test_doctest_skip.txt create mode 100644 Misc/NEWS.d/next/Library/2024-03-27-16-43-42.gh-issue-117294.wbXNFv.rst diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 135758187894ec..a643a0e7e313bf 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1021,7 +1021,8 @@ from text files and modules with doctests: and runs the interactive examples in each file. If an example in any file fails, then the synthesized unit test fails, and a :exc:`failureException` exception is raised showing the name of the file containing the test and a - (sometimes approximate) line number. + (sometimes approximate) line number. If all the examples in a file are + skipped, then the synthesized unit test is also marked as skipped. Pass one or more paths (as strings) to text files to be examined. @@ -1087,7 +1088,8 @@ from text files and modules with doctests: and runs each doctest in the module. If any of the doctests fail, then the synthesized unit test fails, and a :exc:`failureException` exception is raised showing the name of the file containing the test and a (sometimes approximate) - line number. + line number. If all the examples in a docstring are skipped, then the + synthesized unit test is also marked as skipped. Optional argument *module* provides the module to be tested. It can be a module object or a (possibly dotted) module name. If not specified, the module calling diff --git a/Lib/doctest.py b/Lib/doctest.py index 7a9f4e40d814d6..fc0da590018b40 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -2281,12 +2281,13 @@ def runTest(self): try: runner.DIVIDER = "-"*70 - failures, tries = runner.run( - test, out=new.write, clear_globs=False) + results = runner.run(test, out=new.write, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") finally: sys.stdout = old - if failures: + if results.failed: raise self.failureException(self.format_failure(new.getvalue())) def format_failure(self, err): diff --git a/Lib/test/test_doctest/sample_doctest_skip.py b/Lib/test/test_doctest/sample_doctest_skip.py new file mode 100644 index 00000000000000..1b83dec1f8c4dc --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_skip.py @@ -0,0 +1,49 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving skips. +""" + +def no_skip_pass(): + """ + >>> 2 + 2 + 4 + """ + +def no_skip_fail(): + """ + >>> 2 + 2 + 5 + """ + +def single_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + """ + +def double_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 # doctest: +SKIP + 6 + """ + +def partial_skip_pass(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 + 6 + """ + +def partial_skip_fail(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 2 + 2 + 5 + """ + +def no_examples(): + """A docstring with no examples should not be counted as run or skipped.""" diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 3e883c56f6c766..dd8cc9be3a4a8a 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2247,6 +2247,16 @@ def test_DocTestSuite(): >>> suite.run(unittest.TestResult()) + If all examples in a docstring are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_skip') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 2 + We can use the current module: >>> suite = test.test_doctest.sample_doctest.test_suite() @@ -2418,6 +2428,18 @@ def test_DocFileSuite(): Traceback (most recent call last): ValueError: Package may only be specified for module-relative paths. + If all examples in a file are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest4.txt', + ... 'test_doctest_skip.txt') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 1 + You can specify initial global variables: >>> suite = doctest.DocFileSuite('test_doctest.txt', diff --git a/Lib/test/test_doctest/test_doctest_skip.txt b/Lib/test/test_doctest/test_doctest_skip.txt new file mode 100644 index 00000000000000..f340e2b8141253 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip.txt @@ -0,0 +1,4 @@ +This is a sample doctest in a text file, in which all examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 71039d2a8e7ab9..ae8a8c99762313 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -31,7 +31,7 @@ # Retrieve some helpers from other test cases from test.test_doctest import (test_doctest, sample_doctest, sample_doctest_no_doctests, - sample_doctest_no_docstrings) + sample_doctest_no_docstrings, sample_doctest_skip) def _run_object_doctest(obj, module): @@ -110,7 +110,7 @@ def test_doctest_issue4197(self): # The sample doctest files rewritten to include in the zipped version. sample_sources = {} for mod in [sample_doctest, sample_doctest_no_doctests, - sample_doctest_no_docstrings]: + sample_doctest_no_docstrings, sample_doctest_skip]: src = inspect.getsource(mod) src = src.replace("test.test_doctest.test_doctest", "test_zipped_doctest") # Rewrite the module name so that, for example, diff --git a/Misc/NEWS.d/next/Library/2024-03-27-16-43-42.gh-issue-117294.wbXNFv.rst b/Misc/NEWS.d/next/Library/2024-03-27-16-43-42.gh-issue-117294.wbXNFv.rst new file mode 100644 index 00000000000000..bb351e6399a765 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-27-16-43-42.gh-issue-117294.wbXNFv.rst @@ -0,0 +1,2 @@ +A ``DocTestCase`` now reports as skipped if all examples in the doctest are +skipped. From 18cf239e39e25e6cef50ecbb7f197a82f8920ff5 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 28 Mar 2024 14:02:34 -0700 Subject: [PATCH 004/143] Increase the JIT CI timeouts to 75 minutes (GH-117342) --- .github/workflows/jit.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 48c6f555fdc5a0..f18fb0030bbf8b 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -5,13 +5,11 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - - 'Python/optimizer_bytecodes.c' push: paths: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - - 'Python/optimizer_bytecodes.c' workflow_dispatch: concurrency: @@ -22,7 +20,7 @@ jobs: jit: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) runs-on: ${{ matrix.runner }} - timeout-minutes: 60 + timeout-minutes: 75 strategy: fail-fast: false matrix: @@ -95,7 +93,7 @@ jobs: run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} - ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # No PGO or tests (yet): - name: Emulated Windows @@ -111,7 +109,7 @@ jobs: SDKROOT="$(xcrun --show-sdk-path)" \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 - ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - name: Native Linux if: runner.os == 'Linux' && matrix.architecture == 'x86_64' @@ -120,7 +118,7 @@ jobs: export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - name: Emulated Linux if: runner.os == 'Linux' && matrix.architecture != 'x86_64' @@ -140,4 +138,4 @@ jobs: HOSTRUNNER=qemu-${{ matrix.architecture }} \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From 14f1ca7d5363386163839b31ce987423daecc3de Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Thu, 28 Mar 2024 22:20:08 +0100 Subject: [PATCH 005/143] gh-117335: Handle non-iterables for `ntpath.commonpath` (GH-117336) --- Lib/ntpath.py | 11 +++++------ Lib/test/test_ntpath.py | 3 +++ .../2024-03-28-19-13-20.gh-issue-117335.d6uKJu.rst | 1 + 3 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-28-19-13-20.gh-issue-117335.d6uKJu.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index f1c48ecd1e5e2a..ecfc7d48dbb192 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -831,23 +831,22 @@ def relpath(path, start=None): raise -# Return the longest common sub-path of the sequence of paths given as input. +# Return the longest common sub-path of the iterable of paths given as input. # The function is case-insensitive and 'separator-insensitive', i.e. if the # only difference between two paths is the use of '\' versus '/' as separator, # they are deemed to be equal. # # However, the returned path will have the standard '\' separator (even if the # given paths had the alternative '/' separator) and will have the case of the -# first path given in the sequence. Additionally, any trailing separator is +# first path given in the iterable. Additionally, any trailing separator is # stripped from the returned path. def commonpath(paths): - """Given a sequence of path names, returns the longest common sub-path.""" - + """Given an iterable of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) if not paths: - raise ValueError('commonpath() arg is an empty sequence') + raise ValueError('commonpath() arg is an empty iterable') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'\\' altsep = b'/' diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 9cb03e3cd5de8d..c816f99e7e9f1b 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -871,11 +871,14 @@ def check_error(exc, paths): self.assertRaises(exc, ntpath.commonpath, [os.fsencode(p) for p in paths]) + self.assertRaises(TypeError, ntpath.commonpath, None) self.assertRaises(ValueError, ntpath.commonpath, []) + self.assertRaises(ValueError, ntpath.commonpath, iter([])) check_error(ValueError, ['C:\\Program Files', 'Program Files']) check_error(ValueError, ['C:\\Program Files', 'C:Program Files']) check_error(ValueError, ['\\Program Files', 'Program Files']) check_error(ValueError, ['Program Files', 'C:\\Program Files']) + check(['C:\\Program Files'], 'C:\\Program Files') check(['C:\\Program Files', 'C:\\Program Files'], 'C:\\Program Files') check(['C:\\Program Files\\', 'C:\\Program Files'], diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-28-19-13-20.gh-issue-117335.d6uKJu.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-28-19-13-20.gh-issue-117335.d6uKJu.rst new file mode 100644 index 00000000000000..e419b2e97f3886 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-28-19-13-20.gh-issue-117335.d6uKJu.rst @@ -0,0 +1 @@ +Raise TypeError for non-sequences for :func:`ntpath.commonpath`. From 26d328b2ba26374fb8d9ffe8215ecef7c5e3f7a2 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 28 Mar 2024 18:23:08 -0400 Subject: [PATCH 006/143] GH-117121: Add pystats to JIT builds (GH-117346) --- Python/ceval.c | 2 +- Python/ceval_macros.h | 2 ++ Tools/jit/template.c | 7 +++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index cd51011450c3d5..d34db61eecbae2 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -990,7 +990,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #define DPRINTF(level, ...) #endif - OPT_STAT_INC(traces_executed); + ; // dummy statement after a label, before a declaration uint16_t uopcode; #ifdef Py_STATS uint64_t trace_uop_execution_counter = 0; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f2536ed3602c69..1194c11f8ba607 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -392,6 +392,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef _Py_JIT #define GOTO_TIER_TWO(EXECUTOR) \ do { \ + OPT_STAT_INC(traces_executed); \ jit_func jitted = (EXECUTOR)->jit_code; \ next_instr = jitted(frame, stack_pointer, tstate); \ Py_DECREF(tstate->previous_executor); \ @@ -406,6 +407,7 @@ do { \ #else #define GOTO_TIER_TWO(EXECUTOR) \ do { \ + OPT_STAT_INC(traces_executed); \ next_uop = (EXECUTOR)->trace; \ assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); \ goto enter_tier_two; \ diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 9b4fc2af9671eb..f8be4d7f78facd 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -43,6 +43,7 @@ #undef GOTO_TIER_TWO #define GOTO_TIER_TWO(EXECUTOR) \ do { \ + OPT_STAT_INC(traces_executed); \ __attribute__((musttail)) \ return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); \ } while (0) @@ -88,6 +89,10 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) PATCH_VALUE(uint32_t, _target, _JIT_TARGET) PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX) + + OPT_STAT_INC(uops_executed); + UOP_STAT_INC(opcode, execution_count); + // The actual instruction definitions (only one will be used): if (opcode == _JUMP_TO_TOP) { CHECK_EVAL_BREAKER(); @@ -106,9 +111,11 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * GOTO_TIER_ONE(NULL); exit_to_tier1: tstate->previous_executor = (PyObject *)current_executor; + UOP_STAT_INC(opcode, miss); GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); exit_to_trace: { + UOP_STAT_INC(opcode, miss); _PyExitData *exit = ¤t_executor->exits[_exit_index]; Py_INCREF(exit->executor); tstate->previous_executor = (PyObject *)current_executor; From a17f313e3958e825db9a83594c8471a984316536 Mon Sep 17 00:00:00 2001 From: Christopher Chianelli Date: Thu, 28 Mar 2024 18:26:56 -0400 Subject: [PATCH 007/143] gh-117339: Use NULL instead of None for LOAD_SUPER_ATTR in dis docs (GH-117343) --- Doc/library/dis.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 190e994a12cc71..21ac2c87a1859e 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1224,7 +1224,7 @@ iterations of the loop. except that ``namei`` is shifted left by 2 bits instead of 1. The low bit of ``namei`` signals to attempt a method load, as with - :opcode:`LOAD_ATTR`, which results in pushing ``None`` and the loaded method. + :opcode:`LOAD_ATTR`, which results in pushing ``NULL`` and the loaded method. When it is unset a single value is pushed to the stack. The second-low bit of ``namei``, if set, means that this was a two-argument From 8eec7ed714e65d616573b7331780b0aa43c6ed6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=82=85=E7=AB=8B=E4=B8=9A=EF=BC=88Chris=20Fu=EF=BC=89?= <17433201@qq.com> Date: Fri, 29 Mar 2024 08:19:20 +0800 Subject: [PATCH 008/143] gh-117110: Fix subclasses of typing.Any with custom constructors (#117111) --- Lib/test/test_typing.py | 20 +++++++++++++++++++ Lib/typing.py | 2 +- ...-03-21-07-27-36.gh-issue-117110.9K1InX.rst | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-21-07-27-36.gh-issue-117110.9K1InX.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 54c7b976185585..927f74eb69fbc7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -140,6 +140,26 @@ class MockSomething(Something, Mock): pass self.assertIsInstance(ms, Something) self.assertIsInstance(ms, Mock) + def test_subclassing_with_custom_constructor(self): + class Sub(Any): + def __init__(self, *args, **kwargs): pass + # The instantiation must not fail. + Sub(0, s="") + + def test_multiple_inheritance_with_custom_constructors(self): + class Foo: + def __init__(self, x): + self.x = x + + class Bar(Any, Foo): + def __init__(self, x, y): + self.y = y + super().__init__(x) + + b = Bar(1, 2) + self.assertEqual(b.x, 1) + self.assertEqual(b.y, 2) + def test_cannot_instantiate(self): with self.assertRaises(TypeError): Any() diff --git a/Lib/typing.py b/Lib/typing.py index 581d187235dc7e..ef532f6c91539d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -539,7 +539,7 @@ class Any(metaclass=_AnyMeta): def __new__(cls, *args, **kwargs): if cls is Any: raise TypeError("Any cannot be instantiated") - return super().__new__(cls, *args, **kwargs) + return super().__new__(cls) @_SpecialForm diff --git a/Misc/NEWS.d/next/Library/2024-03-21-07-27-36.gh-issue-117110.9K1InX.rst b/Misc/NEWS.d/next/Library/2024-03-21-07-27-36.gh-issue-117110.9K1InX.rst new file mode 100644 index 00000000000000..32f8f81c8d052f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-21-07-27-36.gh-issue-117110.9K1InX.rst @@ -0,0 +1 @@ +Fix a bug that prevents subclasses of :class:`typing.Any` to be instantiated with arguments. Patch by Chris Fu. From 2e9be80c99f635c2f7761e8356b0260922d6e7a6 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 28 Mar 2024 17:58:37 -0700 Subject: [PATCH 009/143] Fix reversed assertRegex checks in test_ssl. (#117351) --- Lib/test/test_ssl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index bd831ac22419af..794944afd66dd0 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -3867,7 +3867,7 @@ def test_min_max_version_mismatch(self): server_hostname=hostname) as s: with self.assertRaises(ssl.SSLError) as e: s.connect((HOST, server.port)) - self.assertRegex("(alert|ALERT)", str(e.exception)) + self.assertRegex(str(e.exception), "(alert|ALERT)") @requires_tls_version('SSLv3') def test_min_max_version_sslv3(self): @@ -4182,7 +4182,7 @@ def cb_raising(ssl_sock, server_name, initial_context): # Allow for flexible libssl error messages. regex = "(SSLV3_ALERT_HANDSHAKE_FAILURE|NO_PRIVATE_VALUE)" - self.assertRegex(regex, cm.exception.reason) + self.assertRegex(cm.exception.reason, regex) self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) def test_sni_callback_wrong_return_type(self): From 7e2fef865899837c47e91ef0180fa59eb03e840b Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 29 Mar 2024 18:40:48 +0900 Subject: [PATCH 010/143] gh-117142: ctypes: Migrate global vars to module state (GH-117189) --- Modules/_ctypes/_ctypes.c | 272 ++++++++++---------- Modules/_ctypes/callbacks.c | 23 +- Modules/_ctypes/callproc.c | 104 ++++---- Modules/_ctypes/cfield.c | 7 +- Modules/_ctypes/ctypes.h | 47 ++-- Modules/_ctypes/stgdict.c | 4 +- Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 +- 7 files changed, 237 insertions(+), 224 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index af094a0fb59e27..6bd1893480027c 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -128,15 +128,6 @@ bytes(cdata) ctypes_state global_state = {0}; -PyObject *PyExc_ArgError = NULL; - -/* This dict maps ctypes types to POINTER types */ -PyObject *_ctypes_ptrtype_cache = NULL; - -/* a callable object used for unpickling: - strong reference to _ctypes._unpickle() function */ -static PyObject *_unpickle; - /****************************************************************/ @@ -208,14 +199,13 @@ static PyType_Spec dictremover_spec = { }; int -PyDict_SetItemProxy(PyObject *dict, PyObject *key, PyObject *item) +PyDict_SetItemProxy(ctypes_state *st, PyObject *dict, PyObject *key, PyObject *item) { PyObject *obj; DictRemoverObject *remover; PyObject *proxy; int result; - ctypes_state *st = GLOBAL_STATE(); obj = _PyObject_CallNoArgs((PyObject *)st->DictRemover_Type); if (obj == NULL) return -1; @@ -562,7 +552,7 @@ static PyType_Spec pyctype_type_spec = { */ static PyCArgObject * -StructUnionType_paramfunc(CDataObject *self) +StructUnionType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; PyObject *obj; @@ -578,7 +568,6 @@ StructUnionType_paramfunc(CDataObject *self) /* Create a Python object which calls PyMem_Free(ptr) in its deallocator. The object will be destroyed at _ctypes_callproc() cleanup. */ - ctypes_state *st = GLOBAL_STATE(); PyTypeObject *tp = st->StructParam_Type; obj = tp->tp_alloc(tp, 0); if (obj == NULL) { @@ -594,13 +583,12 @@ StructUnionType_paramfunc(CDataObject *self) obj = Py_NewRef(self); } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) { Py_DECREF(obj); return NULL; } - ctypes_state *st = GLOBAL_STATE(); StgInfo *stginfo; if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { Py_DECREF(obj); @@ -715,7 +703,8 @@ CDataType_from_address(PyObject *type, PyObject *value) buf = (void *)PyLong_AsVoidPtr(value); if (PyErr_Occurred()) return NULL; - return PyCData_AtAddress(type, buf); + ctypes_state *st = GLOBAL_STATE(); + return PyCData_AtAddress(st, type, buf); } PyDoc_STRVAR(from_buffer_doc, @@ -788,7 +777,7 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return NULL; } - result = PyCData_AtAddress(type, (char *)buffer->buf + offset); + result = PyCData_AtAddress(st, type, (char *)buffer->buf + offset); if (result == NULL) { Py_DECREF(mv); return NULL; @@ -805,6 +794,10 @@ CDataType_from_buffer(PyObject *type, PyObject *args) PyDoc_STRVAR(from_buffer_copy_doc, "C.from_buffer_copy(object, offset=0) -> C instance\ncreate a C instance from a readable buffer"); +static inline PyObject * +generic_pycdata_new(ctypes_state *st, + PyTypeObject *type, PyObject *args, PyObject *kwds); + static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -849,7 +842,7 @@ CDataType_from_buffer_copy(PyObject *type, PyObject *args) return NULL; } - result = GenericPyCData_new((PyTypeObject *)type, NULL, NULL); + result = generic_pycdata_new(st, (PyTypeObject *)type, NULL, NULL); if (result != NULL) { memcpy(((CDataObject *)result)->b_ptr, (char *)buffer.buf + offset, info->size); @@ -917,7 +910,8 @@ CDataType_in_dll(PyObject *type, PyObject *args) return NULL; } #endif - return PyCData_AtAddress(type, address); + ctypes_state *st = GLOBAL_STATE(); + return PyCData_AtAddress(st, type, address); } PyDoc_STRVAR(from_param_doc, @@ -990,7 +984,8 @@ CDataType_repeat(PyObject *self, Py_ssize_t length) return PyErr_Format(PyExc_ValueError, "Array length must be >= 0, not %zd", length); - return PyCArrayType_from_ctype(self, length); + ctypes_state *st = GLOBAL_STATE(); + return PyCArrayType_from_ctype(st, self, length); } static int @@ -1106,9 +1101,8 @@ size property/method, and the sequence protocol. */ static int -PyCPointerType_SetProto(StgInfo *stginfo, PyObject *proto) +PyCPointerType_SetProto(ctypes_state *st, StgInfo *stginfo, PyObject *proto) { - ctypes_state *st = GLOBAL_STATE(); if (!proto || !PyType_Check(proto)) { PyErr_SetString(PyExc_TypeError, "_type_ must be a type"); @@ -1129,11 +1123,11 @@ PyCPointerType_SetProto(StgInfo *stginfo, PyObject *proto) } static PyCArgObject * -PyCPointerType_paramfunc(CDataObject *self) +PyCPointerType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -1176,7 +1170,7 @@ PyCPointerType_init(PyObject *self, PyObject *args, PyObject *kwds) } if (proto) { const char *current_format; - if (-1 == PyCPointerType_SetProto(stginfo, proto)) { + if (PyCPointerType_SetProto(st, stginfo, proto) < 0) { Py_DECREF(proto); return -1; } @@ -1230,7 +1224,7 @@ PyCPointerType_set_type(PyTypeObject *self, PyObject *type) return NULL; } - if (-1 == PyCPointerType_SetProto(info, type)) { + if (PyCPointerType_SetProto(st, info, type) < 0) { Py_DECREF(attrdict); return NULL; } @@ -1244,7 +1238,7 @@ PyCPointerType_set_type(PyTypeObject *self, PyObject *type) Py_RETURN_NONE; } -static PyObject *_byref(PyObject *); +static PyObject *_byref(ctypes_state *, PyObject *); static PyObject * PyCPointerType_from_param(PyObject *type, PyObject *value) @@ -1272,7 +1266,7 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) switch (PyObject_IsInstance(value, typeinfo->proto)) { case 1: Py_INCREF(value); /* _byref steals a refcount */ - return _byref(value); + return _byref(st, value); case -1: return NULL; default: @@ -1506,9 +1500,9 @@ add_getset(PyTypeObject *type, PyGetSetDef *gsp) } static PyCArgObject * -PyCArrayType_paramfunc(CDataObject *self) +PyCArrayType_paramfunc(ctypes_state *st, CDataObject *self) { - PyCArgObject *p = PyCArgObject_new(); + PyCArgObject *p = PyCArgObject_new(st); if (p == NULL) return NULL; p->tag = 'P'; @@ -1684,11 +1678,12 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = GLOBAL_STATE(); if (PyUnicode_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("Z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1706,7 +1701,6 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_wchar array instance or pointer(c_wchar(...)) */ StgInfo *it; @@ -1758,11 +1752,12 @@ c_char_p_from_param(PyObject *type, PyObject *value) if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = GLOBAL_STATE(); if (PyBytes_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1780,7 +1775,6 @@ c_char_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_char array instance or pointer(c_char(...)) */ StgInfo *it; @@ -1834,13 +1828,15 @@ c_void_p_from_param(PyObject *type, PyObject *value) if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = GLOBAL_STATE(); + /* Should probably allow buffer interface as well */ /* int, long */ if (PyLong_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("P"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1858,7 +1854,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1875,7 +1871,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("Z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1895,7 +1891,6 @@ c_void_p_from_param(PyObject *type, PyObject *value) /* c_void_p instances */ return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); /* ctypes array or pointer instance */ if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* Any array or pointer is accepted */ @@ -1914,7 +1909,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; PyCFuncPtrObject *func; func = (PyCFuncPtrObject *)value; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1939,7 +1934,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) switch (PyUnicode_AsUTF8(stgi->proto)[0]) { case 'z': /* c_char_p */ case 'Z': /* c_wchar_p */ - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1969,32 +1964,33 @@ static PyMethodDef c_void_p_method = { "from_param", c_void_p_from_param, METH_O static PyMethodDef c_char_p_method = { "from_param", c_char_p_from_param, METH_O }; static PyMethodDef c_wchar_p_method = { "from_param", c_wchar_p_from_param, METH_O }; -static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject *kwds, +static PyObject *CreateSwappedType(ctypes_state *st, PyTypeObject *type, + PyObject *args, PyObject *kwds, PyObject *proto, struct fielddesc *fmt) { PyTypeObject *result; PyObject *name = PyTuple_GET_ITEM(args, 0); PyObject *newname; PyObject *swapped_args; - static PyObject *suffix; Py_ssize_t i; swapped_args = PyTuple_New(PyTuple_GET_SIZE(args)); if (!swapped_args) return NULL; - if (suffix == NULL) + if (st->swapped_suffix == NULL) { #ifdef WORDS_BIGENDIAN - suffix = PyUnicode_InternFromString("_le"); + st->swapped_suffix = PyUnicode_InternFromString("_le"); #else - suffix = PyUnicode_InternFromString("_be"); + st->swapped_suffix = PyUnicode_InternFromString("_be"); #endif - if (suffix == NULL) { + } + if (st->swapped_suffix == NULL) { Py_DECREF(swapped_args); return NULL; } - newname = PyUnicode_Concat(name, suffix); + newname = PyUnicode_Concat(name, st->swapped_suffix); if (newname == NULL) { Py_DECREF(swapped_args); return NULL; @@ -2014,8 +2010,6 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject if (result == NULL) return NULL; - ctypes_state *st = GLOBAL_STATE(); - StgInfo *stginfo = PyStgInfo_Init(st, result); if (!stginfo) { Py_DECREF(result); @@ -2035,13 +2029,12 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject } static PyCArgObject * -PyCSimpleType_paramfunc(CDataObject *self) +PyCSimpleType_paramfunc(ctypes_state *st, CDataObject *self) { const char *fmt; PyCArgObject *parg; struct fielddesc *fd; - ctypes_state *st = GLOBAL_STATE(); StgInfo *info; if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { return NULL; @@ -2053,7 +2046,7 @@ PyCSimpleType_paramfunc(CDataObject *self) fd = _ctypes_get_fielddesc(fmt); assert(fd); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -2198,7 +2191,7 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) && fmt->setfunc_swapped && fmt->getfunc_swapped) { - PyObject *swapped = CreateSwappedType(type, args, kwds, + PyObject *swapped = CreateSwappedType(st, type, args, kwds, proto, fmt); if (swapped == NULL) { return -1; @@ -2272,7 +2265,7 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) fd = _ctypes_get_fielddesc(fmt); assert(fd); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -2343,7 +2336,7 @@ static PyType_Spec pycsimple_type_spec = { */ static PyObject * -converters_from_argtypes(PyObject *ob) +converters_from_argtypes(ctypes_state *st, PyObject *ob) { PyObject *converters; Py_ssize_t i; @@ -2358,7 +2351,7 @@ converters_from_argtypes(PyObject *ob) Py_ssize_t nArgs = PyTuple_GET_SIZE(ob); if (nArgs > CTYPES_MAX_ARGCOUNT) { Py_DECREF(ob); - PyErr_Format(PyExc_ArgError, + PyErr_Format(st->PyExc_ArgError, "_argtypes_ has too many arguments (%zi), maximum is %i", nArgs, CTYPES_MAX_ARGCOUNT); return NULL; @@ -2444,7 +2437,7 @@ converters_from_argtypes(PyObject *ob) } static int -make_funcptrtype_dict(PyObject *attrdict, StgInfo *stginfo) +make_funcptrtype_dict(ctypes_state *st, PyObject *attrdict, StgInfo *stginfo) { PyObject *ob; PyObject *converters = NULL; @@ -2473,7 +2466,7 @@ make_funcptrtype_dict(PyObject *attrdict, StgInfo *stginfo) return -1; } if (ob) { - converters = converters_from_argtypes(ob); + converters = converters_from_argtypes(st, ob); if (!converters) { Py_DECREF(ob); return -1; @@ -2487,7 +2480,6 @@ make_funcptrtype_dict(PyObject *attrdict, StgInfo *stginfo) } if (ob) { StgInfo *info; - ctypes_state *st = GLOBAL_STATE(); if (PyStgInfo_FromType(st, ob, &info) < 0) { return -1; } @@ -2522,11 +2514,11 @@ make_funcptrtype_dict(PyObject *attrdict, StgInfo *stginfo) } static PyCArgObject * -PyCFuncPtrType_paramfunc(CDataObject *self) +PyCFuncPtrType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -2567,7 +2559,7 @@ PyCFuncPtrType_init(PyObject *self, PyObject *args, PyObject *kwds) } stginfo->flags |= TYPEFLAG_ISPOINTER; - if (-1 == make_funcptrtype_dict(attrdict, stginfo)) { + if (make_funcptrtype_dict(st, attrdict, stginfo) < 0) { Py_DECREF(attrdict); return -1; } @@ -2751,9 +2743,8 @@ static PyMemberDef PyCData_members[] = { /* Find the innermost type of an array type, returning a borrowed reference */ static PyObject * -PyCData_item_type(PyObject *type) +PyCData_item_type(ctypes_state *st, PyObject *type) { - ctypes_state *st = GLOBAL_STATE(); if (PyCArrayTypeObject_Check(st, type)) { PyObject *elem_type; @@ -2765,7 +2756,7 @@ PyCData_item_type(PyObject *type) assert(stg_info); elem_type = stg_info->proto; assert(elem_type); - return PyCData_item_type(elem_type); + return PyCData_item_type(st, elem_type); } else { return type; @@ -2784,7 +2775,7 @@ PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) } assert(info); - PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); + PyObject *item_type = PyCData_item_type(st, (PyObject*)Py_TYPE(myself)); if (item_type == NULL) { return 0; } @@ -2843,7 +2834,7 @@ PyCData_reduce(PyObject *myself, PyObject *args) if (dict == NULL) { return NULL; } - return Py_BuildValue("O(O(NN))", _unpickle, Py_TYPE(myself), dict, + return Py_BuildValue("O(O(NN))", st->_unpickle, Py_TYPE(myself), dict, PyBytes_FromStringAndSize(self->b_ptr, self->b_size)); } @@ -2947,13 +2938,13 @@ PyCData_MallocBuffer(CDataObject *obj, StgInfo *info) } PyObject * -PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) +PyCData_FromBaseObj(ctypes_state *st, + PyObject *type, PyObject *base, Py_ssize_t index, char *adr) { CDataObject *cmem; assert(PyType_Check(type)); - ctypes_state *st = GLOBAL_STATE(); StgInfo *info; if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; @@ -2969,11 +2960,11 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) if (cmem == NULL) { return NULL; } - assert(CDataObject_Check(GLOBAL_STATE(), cmem)); + assert(CDataObject_Check(st, cmem)); cmem->b_length = info->length; cmem->b_size = info->size; if (base) { /* use base's buffer */ - assert(CDataObject_Check(GLOBAL_STATE(), base)); + assert(CDataObject_Check(st, base)); cmem->b_ptr = adr; cmem->b_needsfree = 0; cmem->b_base = (CDataObject *)Py_NewRef(base); @@ -2993,7 +2984,7 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) Box a memory block into a CData instance. */ PyObject * -PyCData_AtAddress(PyObject *type, void *buf) +PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf) { CDataObject *pd; @@ -3003,7 +2994,6 @@ PyCData_AtAddress(PyObject *type, void *buf) assert(PyType_Check(type)); - ctypes_state *st = GLOBAL_STATE(); StgInfo *info; if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; @@ -3020,7 +3010,7 @@ PyCData_AtAddress(PyObject *type, void *buf) if (!pd) { return NULL; } - assert(CDataObject_Check(GLOBAL_STATE(), pd)); + assert(CDataObject_Check(st, pd)); pd->b_ptr = (char *)buf; pd->b_length = info->length; pd->b_size = info->size; @@ -3032,10 +3022,9 @@ PyCData_AtAddress(PyObject *type, void *buf) classes. FALSE otherwise FALSE also for subclasses of c_int and such. */ -int _ctypes_simple_instance(PyObject *obj) +int _ctypes_simple_instance(ctypes_state *st, PyObject *obj) { PyTypeObject *type = (PyTypeObject *)obj; - ctypes_state *st = GLOBAL_STATE(); if (PyCSimpleTypeObject_Check(st, type)) { return type->tp_base != st->Simple_Type; @@ -3044,27 +3033,28 @@ int _ctypes_simple_instance(PyObject *obj) } PyObject * -PyCData_get(PyObject *type, GETFUNC getfunc, PyObject *src, +PyCData_get(ctypes_state *st, PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *adr) { if (getfunc) return getfunc(adr, size); assert(type); - ctypes_state *st = GLOBAL_STATE(); StgInfo *info; if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; } - if (info && info->getfunc && !_ctypes_simple_instance(type)) + if (info && info->getfunc && !_ctypes_simple_instance(st, type)) { return info->getfunc(adr, size); - return PyCData_FromBaseObj(type, src, index, adr); + } + return PyCData_FromBaseObj(st, type, src, index, adr); } /* Helper function for PyCData_set below. */ static PyObject * -_PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +_PyCData_set(ctypes_state *st, + CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t size, char *ptr) { CDataObject *src; @@ -3073,7 +3063,6 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, if (setfunc) { return setfunc(ptr, value, size); } - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, value)) { StgInfo *info; if (PyStgInfo_FromType(st, type, &info) < 0) { @@ -3095,7 +3084,7 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, ((PyTypeObject *)type)->tp_name); return NULL; } - result = _PyCData_set(dst, type, setfunc, ob, + result = _PyCData_set(st, dst, type, setfunc, ob, size, ptr); Py_DECREF(ob); return result; @@ -3180,12 +3169,12 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, * to the value 'value'. */ int -PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +PyCData_set(ctypes_state *st, + PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr) { CDataObject *mem = (CDataObject *)dst; PyObject *result; - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, dst)) { PyErr_SetString(PyExc_TypeError, @@ -3193,7 +3182,7 @@ PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, return -1; } - result = _PyCData_set(mem, type, setfunc, value, + result = _PyCData_set(st, mem, type, setfunc, value, size, ptr); if (result == NULL) return -1; @@ -3208,10 +3197,17 @@ PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, /******************************************************************/ static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + ctypes_state *st = GLOBAL_STATE(); + return generic_pycdata_new(st, type, args, kwds); +} + +static inline PyObject * +generic_pycdata_new(ctypes_state *st, + PyTypeObject *type, PyObject *args, PyObject *kwds) { CDataObject *obj; - ctypes_state *st = GLOBAL_STATE(); StgInfo *info; if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { return NULL; @@ -3326,7 +3322,8 @@ PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ig Py_CLEAR(self->converters); Py_CLEAR(self->argtypes); } else { - converters = converters_from_argtypes(ob); + ctypes_state *st = GLOBAL_STATE(); + converters = converters_from_argtypes(st, ob); if (!converters) return -1; Py_XSETREF(self->converters, converters); @@ -3422,10 +3419,8 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) /* Return 1 if usable, 0 else and exception set. */ static int -_check_outarg_type(PyObject *arg, Py_ssize_t index) +_check_outarg_type(ctypes_state *st, PyObject *arg, Py_ssize_t index) { - ctypes_state *st = GLOBAL_STATE(); - if (PyCPointerTypeObject_Check(st, arg)) { return 1; } @@ -3455,12 +3450,11 @@ _check_outarg_type(PyObject *arg, Py_ssize_t index) /* Returns 1 on success, 0 on error */ static int -_validate_paramflags(PyTypeObject *type, PyObject *paramflags) +_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) { Py_ssize_t i, len; PyObject *argtypes; - ctypes_state *st = GLOBAL_STATE(); StgInfo *info; if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { return -1; @@ -3509,7 +3503,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) case PARAMFLAG_FIN | PARAMFLAG_FOUT: break; case PARAMFLAG_FOUT: - if (!_check_outarg_type(typ, i+1)) + if (!_check_outarg_type(st, typ, i+1)) return 0; break; default: @@ -3641,12 +3635,13 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #endif - if (!_validate_paramflags(type, paramflags)) { + ctypes_state *st = GLOBAL_STATE(); + if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); return NULL; } - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); if (!self) { Py_DECREF(ftuple); return NULL; @@ -3682,10 +3677,11 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) if (paramflags == Py_None) paramflags = NULL; - if (!_validate_paramflags(type, paramflags)) + ctypes_state *st = GLOBAL_STATE(); + if (!_validate_paramflags(st, type, paramflags)) { return NULL; - - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + } + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); self->index = index + 0x1000; self->paramflags = Py_XNewRef(paramflags); if (iid_len == sizeof(GUID)) @@ -3775,14 +3771,15 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } - thunk = _ctypes_alloc_callback(callable, + thunk = _ctypes_alloc_callback(st, + callable, info->argtypes, info->restype, info->flags); if (!thunk) return NULL; - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); if (self == NULL) { Py_DECREF(thunk); return NULL; @@ -3806,10 +3803,9 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _byref consumes a refcount to its argument */ static PyObject * -_byref(PyObject *obj) +_byref(ctypes_state *st, PyObject *obj) { PyCArgObject *parg; - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, @@ -3817,7 +3813,7 @@ _byref(PyObject *obj) return NULL; } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) { Py_DECREF(obj); return NULL; @@ -3881,7 +3877,7 @@ _get_arg(int *pindex, PyObject *name, PyObject *defval, PyObject *inargs, PyObje function. */ static PyObject * -_build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, +_build_callargs(ctypes_state *st, PyCFuncPtrObject *self, PyObject *argtypes, PyObject *inargs, PyObject *kwds, int *poutmask, int *pinoutmask, unsigned int *pnumretvals) { @@ -3918,7 +3914,6 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, inargs_index = 1; } #endif - ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < len; ++i) { PyObject *item = PyTuple_GET_ITEM(paramflags, i); PyObject *ob; @@ -4156,7 +4151,6 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) "native com method call without 'this' parameter"); return NULL; } - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, this)) { PyErr_SetString(PyExc_TypeError, "Expected a COM this pointer as first argument"); @@ -4178,7 +4172,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) pProc = ((void **)piunk->lpVtbl)[self->index - 0x1000]; } #endif - callargs = _build_callargs(self, argtypes, + callargs = _build_callargs(st, self, argtypes, inargs, kwds, &outmask, &inoutmask, &numretvals); if (callargs == NULL) @@ -4214,7 +4208,8 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) } } - result = _ctypes_callproc(pProc, + result = _ctypes_callproc(st, + pProc, callargs, #ifdef MS_WIN32 piunk, @@ -4536,7 +4531,7 @@ Array_item(PyObject *myself, Py_ssize_t index) size = stginfo->size / stginfo->length; offset = index * size; - return PyCData_get(stginfo->proto, stginfo->getfunc, (PyObject *)self, + return PyCData_get(st, stginfo->proto, stginfo->getfunc, (PyObject *)self, index, size, self->b_ptr + offset); } @@ -4682,7 +4677,7 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) offset = index * size; ptr = self->b_ptr + offset; - return PyCData_set((PyObject *)self, stginfo->proto, stginfo->setfunc, value, + return PyCData_set(st, (PyObject *)self, stginfo->proto, stginfo->setfunc, value, index, size, ptr); } @@ -4789,17 +4784,17 @@ static PyType_Spec pycarray_spec = { }; PyObject * -PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) +PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length) { - static PyObject *cache; PyObject *key; char name[256]; PyObject *len; - if (cache == NULL) { - cache = PyDict_New(); - if (cache == NULL) + if (st->array_cache == NULL) { + st->array_cache = PyDict_New(); + if (st->array_cache == NULL) { return NULL; + } } len = PyLong_FromSsize_t(length); if (len == NULL) @@ -4810,7 +4805,7 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) return NULL; PyObject *result; - if (_PyDict_GetItemProxy(cache, key, &result) != 0) { + if (_PyDict_GetItemProxy(st->array_cache, key, &result) != 0) { // found or error Py_DECREF(key); return result; @@ -4829,7 +4824,6 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) sprintf(name, "%.200s_Array_%ld", ((PyTypeObject *)itemtype)->tp_name, (long)length); #endif - ctypes_state *st = GLOBAL_STATE(); result = PyObject_CallFunction((PyObject *)st->PyCArrayType_Type, "s(O){s:n,s:O}", name, @@ -4843,7 +4837,7 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) Py_DECREF(key); return NULL; } - if (-1 == PyDict_SetItemProxy(cache, key, result)) { + if (PyDict_SetItemProxy(st, st->array_cache, key, result) < 0) { Py_DECREF(key); Py_DECREF(result); return NULL; @@ -4918,7 +4912,8 @@ static PyGetSetDef Simple_getsets[] = { static PyObject * Simple_from_outparm(PyObject *self, PyObject *args) { - if (_ctypes_simple_instance((PyObject *)Py_TYPE(self))) { + ctypes_state *st = GLOBAL_STATE(); + if (_ctypes_simple_instance(st, (PyObject *)Py_TYPE(self))) { return Py_NewRef(self); } /* call stginfo->getfunc */ @@ -5015,7 +5010,7 @@ Pointer_item(PyObject *myself, Py_ssize_t index) size = iteminfo->size; offset = index * iteminfo->size; - return PyCData_get(proto, stginfo->getfunc, (PyObject *)self, + return PyCData_get(st, proto, stginfo->getfunc, (PyObject *)self, index, size, (*(char **)self->b_ptr) + offset); } @@ -5059,7 +5054,7 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) size = iteminfo->size; offset = index * iteminfo->size; - return PyCData_set((PyObject *)self, proto, stginfo->setfunc, value, + return PyCData_set(st, (PyObject *)self, proto, stginfo->setfunc, value, index, size, (*(char **)self->b_ptr) + offset); } @@ -5079,7 +5074,7 @@ Pointer_get_contents(CDataObject *self, void *closure) } assert(stginfo); /* Cannot be NULL for pointer instances */ - return PyCData_FromBaseObj(stginfo->proto, + return PyCData_FromBaseObj(st, stginfo->proto, (PyObject *)self, 0, *(void **)self->b_ptr); } @@ -5167,7 +5162,7 @@ Pointer_new(PyTypeObject *type, PyObject *args, PyObject *kw) "Cannot create instance: has no _type_"); return NULL; } - return GenericPyCData_new(type, args, kw); + return generic_pycdata_new(st, type, args, kw); } static PyObject * @@ -5436,10 +5431,8 @@ string_at(const char *ptr, int size) } static int -cast_check_pointertype(PyObject *arg) +cast_check_pointertype(ctypes_state *st, PyObject *arg) { - ctypes_state *st = GLOBAL_STATE(); - if (PyCPointerTypeObject_Check(st, arg)) { return 1; } @@ -5468,9 +5461,12 @@ cast_check_pointertype(PyObject *arg) static PyObject * cast(void *ptr, PyObject *src, PyObject *ctype) { + ctypes_state *st = GLOBAL_STATE(); + CDataObject *result; - if (0 == cast_check_pointertype(ctype)) + if (cast_check_pointertype(st, ctype) == 0) { return NULL; + } result = (CDataObject *)_PyObject_CallNoArgs(ctype); if (result == NULL) return NULL; @@ -5481,7 +5477,6 @@ cast(void *ptr, PyObject *src, PyObject *ctype) It must certainly contain the source objects one. It must contain the source object itself. */ - ctypes_state *st = GLOBAL_STATE(); if (CDataObject_Check(st, src)) { CDataObject *obj = (CDataObject *)src; CDataObject *container; @@ -5652,10 +5647,10 @@ _ctypes_add_objects(PyObject *mod) } \ } while (0) - MOD_ADD("_pointer_type_cache", Py_NewRef(_ctypes_ptrtype_cache)); + ctypes_state *st = GLOBAL_STATE(); + MOD_ADD("_pointer_type_cache", Py_NewRef(st->_ctypes_ptrtype_cache)); #ifdef MS_WIN32 - ctypes_state *st = GLOBAL_STATE(); MOD_ADD("COMError", Py_NewRef(st->PyComError_Type)); MOD_ADD("FUNCFLAG_HRESULT", PyLong_FromLong(FUNCFLAG_HRESULT)); MOD_ADD("FUNCFLAG_STDCALL", PyLong_FromLong(FUNCFLAG_STDCALL)); @@ -5685,7 +5680,7 @@ _ctypes_add_objects(PyObject *mod) MOD_ADD("RTLD_LOCAL", PyLong_FromLong(RTLD_LOCAL)); MOD_ADD("RTLD_GLOBAL", PyLong_FromLong(RTLD_GLOBAL)); MOD_ADD("CTYPES_MAX_ARGCOUNT", PyLong_FromLong(CTYPES_MAX_ARGCOUNT)); - MOD_ADD("ArgumentError", Py_NewRef(PyExc_ArgError)); + MOD_ADD("ArgumentError", Py_NewRef(st->PyExc_ArgError)); MOD_ADD("SIZEOF_TIME_T", PyLong_FromSsize_t(SIZEOF_TIME_T)); return 0; #undef MOD_ADD @@ -5695,18 +5690,19 @@ _ctypes_add_objects(PyObject *mod) static int _ctypes_mod_exec(PyObject *mod) { - _unpickle = PyObject_GetAttrString(mod, "_unpickle"); - if (_unpickle == NULL) { + ctypes_state *st = GLOBAL_STATE(); + st->_unpickle = PyObject_GetAttrString(mod, "_unpickle"); + if (st->_unpickle == NULL) { return -1; } - _ctypes_ptrtype_cache = PyDict_New(); - if (_ctypes_ptrtype_cache == NULL) { + st->_ctypes_ptrtype_cache = PyDict_New(); + if (st->_ctypes_ptrtype_cache == NULL) { return -1; } - PyExc_ArgError = PyErr_NewException("ctypes.ArgumentError", NULL, NULL); - if (!PyExc_ArgError) { + st->PyExc_ArgError = PyErr_NewException("ctypes.ArgumentError", NULL, NULL); + if (!st->PyExc_ArgError) { return -1; } diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 08d068e47ee2bf..b6f98e92e1ba88 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -136,7 +136,8 @@ TryAddRef(PyObject *cnv, CDataObject *obj) * Call the python object with all arguments * */ -static void _CallPythonObject(void *mem, +static void _CallPythonObject(ctypes_state *st, + void *mem, ffi_type *restype, SETFUNC setfunc, PyObject *callable, @@ -155,7 +156,6 @@ static void _CallPythonObject(void *mem, assert(nargs <= CTYPES_MAX_ARGCOUNT); PyObject **args = alloca(nargs * sizeof(PyObject *)); PyObject **cnvs = PySequence_Fast_ITEMS(converters); - ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < nargs; i++) { PyObject *cnv = cnvs[i]; // borrowed ref @@ -164,7 +164,7 @@ static void _CallPythonObject(void *mem, goto Done; } - if (info && info->getfunc && !_ctypes_simple_instance(cnv)) { + if (info && info->getfunc && !_ctypes_simple_instance(st, cnv)) { PyObject *v = info->getfunc(*pArgs, info->size); if (!v) { PrintError("create argument %zd:\n", i); @@ -205,7 +205,7 @@ static void _CallPythonObject(void *mem, } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { - error_object = _ctypes_get_errobj(&space); + error_object = _ctypes_get_errobj(st, &space); if (error_object == NULL) goto Done; if (flags & FUNCFLAG_USE_ERRNO) { @@ -303,8 +303,10 @@ static void closure_fcn(ffi_cif *cif, void *userdata) { CThunkObject *p = (CThunkObject *)userdata; + ctypes_state *st = GLOBAL_STATE(); - _CallPythonObject(resp, + _CallPythonObject(st, + resp, p->ffi_restype, p->setfunc, p->callable, @@ -313,12 +315,11 @@ static void closure_fcn(ffi_cif *cif, args); } -static CThunkObject* CThunkObject_new(Py_ssize_t nargs) +static CThunkObject* CThunkObject_new(ctypes_state *st, Py_ssize_t nargs) { CThunkObject *p; Py_ssize_t i; - ctypes_state *st = GLOBAL_STATE(); p = PyObject_GC_NewVar(CThunkObject, st->PyCThunk_Type, nargs); if (p == NULL) { return NULL; @@ -340,7 +341,8 @@ static CThunkObject* CThunkObject_new(Py_ssize_t nargs) return p; } -CThunkObject *_ctypes_alloc_callback(PyObject *callable, +CThunkObject *_ctypes_alloc_callback(ctypes_state *st, + PyObject *callable, PyObject *converters, PyObject *restype, int flags) @@ -352,11 +354,10 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(PyTuple_Check(converters)); nargs = PyTuple_GET_SIZE(converters); - p = CThunkObject_new(nargs); + p = CThunkObject_new(st, nargs); if (p == NULL) return NULL; - ctypes_state *st = GLOBAL_STATE(); assert(CThunk_CheckExact(st, (PyObject *)p)); p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); @@ -369,7 +370,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, PyObject **cnvs = PySequence_Fast_ITEMS(converters); for (i = 0; i < nargs; ++i) { PyObject *cnv = cnvs[i]; // borrowed ref - p->atypes[i] = _ctypes_get_ffi_type(cnv); + p->atypes[i] = _ctypes_get_ffi_type(st, cnv); } p->atypes[i] = NULL; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 6ebbb64d61b07a..67d6ade43a2667 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -153,22 +153,22 @@ static void pymem_destructor(PyObject *ptr) kept alive in the thread state dictionary as long as the thread itself. */ PyObject * -_ctypes_get_errobj(int **pspace) +_ctypes_get_errobj(ctypes_state *st, int **pspace) { PyObject *dict = PyThreadState_GetDict(); PyObject *errobj; - static PyObject *error_object_name; if (dict == NULL) { PyErr_SetString(PyExc_RuntimeError, "cannot get thread state"); return NULL; } - if (error_object_name == NULL) { - error_object_name = PyUnicode_InternFromString("ctypes.error_object"); - if (error_object_name == NULL) + if (st->error_object_name == NULL) { + st->error_object_name = PyUnicode_InternFromString("ctypes.error_object"); + if (st->error_object_name == NULL) { return NULL; + } } - if (PyDict_GetItemRef(dict, error_object_name, &errobj) < 0) { + if (PyDict_GetItemRef(dict, st->error_object_name, &errobj) < 0) { return NULL; } if (errobj) { @@ -188,8 +188,7 @@ _ctypes_get_errobj(int **pspace) PyMem_Free(space); return NULL; } - if (-1 == PyDict_SetItem(dict, error_object_name, - errobj)) { + if (PyDict_SetItem(dict, st->error_object_name, errobj) < 0) { Py_DECREF(errobj); return NULL; } @@ -202,7 +201,8 @@ static PyObject * get_error_internal(PyObject *self, PyObject *args, int index) { int *space; - PyObject *errobj = _ctypes_get_errobj(&space); + ctypes_state *st = GLOBAL_STATE(); + PyObject *errobj = _ctypes_get_errobj(st, &space); PyObject *result; if (errobj == NULL) @@ -222,7 +222,8 @@ set_error_internal(PyObject *self, PyObject *args, int index) if (!PyArg_ParseTuple(args, "i", &new_errno)) { return NULL; } - errobj = _ctypes_get_errobj(&space); + ctypes_state *st = GLOBAL_STATE(); + errobj = _ctypes_get_errobj(st, &space); if (errobj == NULL) return NULL; old_errno = space[index]; @@ -473,10 +474,9 @@ check_hresult(PyObject *self, PyObject *args) /**************************************************************/ PyCArgObject * -PyCArgObject_new(void) +PyCArgObject_new(ctypes_state *st) { PyCArgObject *p; - ctypes_state *st = GLOBAL_STATE(); p = PyObject_GC_New(PyCArgObject, st->PyCArg_Type); if (p == NULL) return NULL; @@ -662,10 +662,10 @@ struct argument { /* * Convert a single Python object into a PyCArgObject and return it. */ -static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) +static int ConvParam(ctypes_state *st, + PyObject *obj, Py_ssize_t index, struct argument *pa) { pa->keep = NULL; /* so we cannot forget it later */ - ctypes_state *st = GLOBAL_STATE(); StgInfo *info; int result = PyStgInfo_FromObject(st, obj, &info); @@ -677,7 +677,7 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) PyCArgObject *carg; assert(info->paramfunc); /* If it has an stginfo, it is a CDataObject */ - carg = info->paramfunc((CDataObject *)obj); + carg = info->paramfunc(st, (CDataObject *)obj); if (carg == NULL) return -1; pa->ffi_type = carg->pffi_type; @@ -748,7 +748,7 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) */ if (arg) { int result; - result = ConvParam(arg, index, pa); + result = ConvParam(st, arg, index, pa); Py_DECREF(arg); return result; } @@ -783,13 +783,12 @@ int can_return_struct_as_sint64(size_t s) // returns NULL with exception set on error -ffi_type *_ctypes_get_ffi_type(PyObject *obj) +ffi_type *_ctypes_get_ffi_type(ctypes_state *st, PyObject *obj) { if (obj == NULL) { return &ffi_type_sint; } - ctypes_state *st = GLOBAL_STATE(); StgInfo *info; if (PyStgInfo_FromType(st, obj, &info) < 0) { return NULL; @@ -825,7 +824,8 @@ ffi_type *_ctypes_get_ffi_type(PyObject *obj) * * void ffi_call(ffi_cif *cif, void *fn, void *rvalue, void **avalues); */ -static int _call_function_pointer(int flags, +static int _call_function_pointer(ctypes_state *st, + int flags, PPROC pProc, void **avalues, ffi_type **atypes, @@ -926,7 +926,7 @@ static int _call_function_pointer(int flags, } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { - error_object = _ctypes_get_errobj(&space); + error_object = _ctypes_get_errobj(st, &space); if (error_object == NULL) return -1; } @@ -993,7 +993,8 @@ static int _call_function_pointer(int flags, * - If restype is another ctypes type, return an instance of that. * - Otherwise, call restype and return the result. */ -static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) +static PyObject *GetResult(ctypes_state *st, + PyObject *restype, void *result, PyObject *checker) { PyObject *retval, *v; @@ -1004,7 +1005,6 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) Py_RETURN_NONE; } - ctypes_state *st = GLOBAL_STATE(); StgInfo *info; if (PyStgInfo_FromType(st, restype, &info) < 0) { return NULL; @@ -1013,7 +1013,7 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) return PyObject_CallFunction(restype, "i", *(int *)result); } - if (info->getfunc && !_ctypes_simple_instance(restype)) { + if (info->getfunc && !_ctypes_simple_instance(st, restype)) { retval = info->getfunc(result, info->size); /* If restype is py_object (detected by comparing getfunc with O_get), we have to call Py_DECREF because O_get has already @@ -1022,9 +1022,10 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) if (info->getfunc == _ctypes_get_fielddesc("O")->getfunc) { Py_DECREF(retval); } - } else - retval = PyCData_FromBaseObj(restype, NULL, 0, result); - + } + else { + retval = PyCData_FromBaseObj(st, restype, NULL, 0, result); + } if (!checker || !retval) return retval; @@ -1086,7 +1087,7 @@ void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) #ifdef MS_WIN32 static PyObject * -GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) +GetComError(ctypes_state *st, HRESULT errcode, GUID *riid, IUnknown *pIunk) { HRESULT hr; ISupportErrorInfo *psei = NULL; @@ -1138,7 +1139,6 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) descr, source, helpfile, helpcontext, progid); if (obj) { - ctypes_state *st = GLOBAL_STATE(); PyErr_SetObject((PyObject *)st->PyComError_Type, obj); Py_DECREF(obj); } @@ -1169,7 +1169,8 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) * * - XXX various requirements for restype, not yet collected */ -PyObject *_ctypes_callproc(PPROC pProc, +PyObject *_ctypes_callproc(ctypes_state *st, + PPROC pProc, PyObject *argtuple, #ifdef MS_WIN32 IUnknown *pIunk, @@ -1199,7 +1200,7 @@ PyObject *_ctypes_callproc(PPROC pProc, if (argcount > CTYPES_MAX_ARGCOUNT) { - PyErr_Format(PyExc_ArgError, "too many arguments (%zi), maximum is %i", + PyErr_Format(st->PyExc_ArgError, "too many arguments (%zi), maximum is %i", argcount, CTYPES_MAX_ARGCOUNT); return NULL; } @@ -1232,20 +1233,20 @@ PyObject *_ctypes_callproc(PPROC pProc, converter = PyTuple_GET_ITEM(argtypes, i); v = PyObject_CallOneArg(converter, arg); if (v == NULL) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; } - err = ConvParam(v, i+1, pa); + err = ConvParam(st, v, i+1, pa); Py_DECREF(v); if (-1 == err) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; } } else { - err = ConvParam(arg, i+1, pa); + err = ConvParam(st, arg, i+1, pa); if (-1 == err) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; /* leaking ? */ } } @@ -1254,7 +1255,7 @@ PyObject *_ctypes_callproc(PPROC pProc, if (restype == Py_None) { rtype = &ffi_type_void; } else { - rtype = _ctypes_get_ffi_type(restype); + rtype = _ctypes_get_ffi_type(st, restype); } if (!rtype) { goto cleanup; @@ -1296,7 +1297,7 @@ PyObject *_ctypes_callproc(PPROC pProc, avalues[i] = (void *)&args[i].value; } - if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, + if (-1 == _call_function_pointer(st, flags, pProc, avalues, atypes, rtype, resbuf, Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) @@ -1324,7 +1325,7 @@ PyObject *_ctypes_callproc(PPROC pProc, #ifdef MS_WIN32 if (iid && pIunk) { if (*(int *)resbuf & 0x80000000) - retval = GetComError(*(HRESULT *)resbuf, iid, pIunk); + retval = GetComError(st, *(HRESULT *)resbuf, iid, pIunk); else retval = PyLong_FromLong(*(int *)resbuf); } else if (flags & FUNCFLAG_HRESULT) { @@ -1334,7 +1335,7 @@ PyObject *_ctypes_callproc(PPROC pProc, retval = PyLong_FromLong(*(int *)resbuf); } else #endif - retval = GetResult(restype, resbuf, checker); + retval = GetResult(st, restype, resbuf, checker); cleanup: for (i = 0; i < argcount; ++i) Py_XDECREF(args[i].keep); @@ -1463,8 +1464,10 @@ copy_com_pointer(PyObject *self, PyObject *args) return NULL; a.keep = b.keep = NULL; - if (-1 == ConvParam(p1, 0, &a) || -1 == ConvParam(p2, 1, &b)) + ctypes_state *st = GLOBAL_STATE(); + if (ConvParam(st, p1, 0, &a) < 0 || ConvParam(st, p2, 1, &b) < 0) { goto done; + } src = (IUnknown *)a.value.p; pdst = (IUnknown **)b.value.p; @@ -1643,7 +1646,9 @@ call_function(PyObject *self, PyObject *args) return NULL; } - result = _ctypes_callproc((PPROC)func, + ctypes_state *st = GLOBAL_STATE(); + result = _ctypes_callproc(st, + (PPROC)func, arguments, #ifdef MS_WIN32 NULL, @@ -1678,7 +1683,9 @@ call_cdeclfunction(PyObject *self, PyObject *args) return NULL; } - result = _ctypes_callproc((PPROC)func, + ctypes_state *st = GLOBAL_STATE(); + result = _ctypes_callproc(st, + (PPROC)func, arguments, #ifdef MS_WIN32 NULL, @@ -1728,7 +1735,7 @@ PyDoc_STRVAR(alignment_doc, static PyObject * align_func(PyObject *self, PyObject *obj) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = GLOBAL_STATE(); StgInfo *info; if (PyStgInfo_FromAny(st, obj, &info) < 0) { return NULL; @@ -1774,7 +1781,7 @@ byref(PyObject *self, PyObject *args) return NULL; } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -1949,11 +1956,11 @@ create_pointer_type(PyObject *module, PyObject *cls) PyTypeObject *typ; PyObject *key; - if (PyDict_GetItemRef(_ctypes_ptrtype_cache, cls, &result) != 0) { + ctypes_state *st = GLOBAL_STATE(); + if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, cls, &result) != 0) { // found or error return result; } - ctypes_state *st = GLOBAL_STATE(); // not found if (PyUnicode_CheckExact(cls)) { PyObject *name = PyUnicode_FromFormat("LP_%U", cls); @@ -1983,7 +1990,7 @@ create_pointer_type(PyObject *module, PyObject *cls) PyErr_SetString(PyExc_TypeError, "must be a ctypes type"); return NULL; } - if (-1 == PyDict_SetItem(_ctypes_ptrtype_cache, key, result)) { + if (PyDict_SetItem(st->_ctypes_ptrtype_cache, key, result) < 0) { Py_DECREF(result); Py_DECREF(key); return NULL; @@ -2012,7 +2019,8 @@ create_pointer_inst(PyObject *module, PyObject *arg) PyObject *result; PyObject *typ; - if (PyDict_GetItemRef(_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { + ctypes_state *st = GLOBAL_STATE(); + if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { return NULL; } if (typ == NULL) { diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 16b66382bfe33f..ffe00e25aff49f 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -44,7 +44,7 @@ static void pymem_destructor(PyObject *ptr) * prev_desc points to the type of the previous bitfield, if any. */ PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, +PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, int *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian) @@ -60,7 +60,6 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, #define CONT_BITFIELD 2 #define EXPAND_BITFIELD 3 - ctypes_state *st = GLOBAL_STATE(); PyTypeObject *tp = st->PyCField_Type; self = (CFieldObject *)tp->tp_alloc(tp, 0); if (self == NULL) @@ -230,7 +229,7 @@ PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) "can't delete attribute"); return -1; } - return PyCData_set(inst, self->proto, self->setfunc, value, + return PyCData_set(st, inst, self->proto, self->setfunc, value, self->index, self->size, ptr); } @@ -248,7 +247,7 @@ PyCField_get(CFieldObject *self, PyObject *inst, PyTypeObject *type) return NULL; } src = (CDataObject *)inst; - return PyCData_get(self->proto, self->getfunc, inst, + return PyCData_get(st, self->proto, self->getfunc, inst, self->index, self->size, src->b_ptr + self->offset); } diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index d7d725a4fdf669..3422310045bcc9 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -42,6 +42,7 @@ typedef struct { PyTypeObject *PyCField_Type; PyTypeObject *PyCThunk_Type; PyTypeObject *StructParam_Type; + PyTypeObject *PyCType_Type; PyTypeObject *PyCStructType_Type; PyTypeObject *UnionType_Type; PyTypeObject *PyCPointerType_Type; @@ -58,7 +59,15 @@ typedef struct { #ifdef MS_WIN32 PyTypeObject *PyComError_Type; #endif - PyTypeObject *PyCType_Type; + /* This dict maps ctypes types to POINTER types */ + PyObject *_ctypes_ptrtype_cache; + /* a callable object used for unpickling: + strong reference to _ctypes._unpickle() function */ + PyObject *_unpickle; + PyObject *array_cache; + PyObject *error_object_name; // callproc.c + PyObject *PyExc_ArgError; + PyObject *swapped_suffix; } ctypes_state; extern ctypes_state global_state; @@ -73,7 +82,7 @@ typedef struct tagPyCArgObject PyCArgObject; typedef struct tagCDataObject CDataObject; typedef PyObject *(* GETFUNC)(void *, Py_ssize_t size); typedef PyObject *(* SETFUNC)(void *, PyObject *value, Py_ssize_t size); -typedef PyCArgObject *(* PARAMFUNC)(CDataObject *obj); +typedef PyCArgObject *(* PARAMFUNC)(ctypes_state *st, CDataObject *obj); /* A default buffer in CDataObject, which can be used for small C types. If this buffer is too small, PyMem_Malloc will be called to create a larger one, @@ -175,13 +184,13 @@ extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); extern PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, +PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, int *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int is_big_endian); -extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); -extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length); +extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); +extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); #define PyCArrayTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArrayType_Type) #define ArrayObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArray_Type) @@ -192,11 +201,12 @@ extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length #define PyCStructTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCStructType_Type) extern PyObject * -PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length); +PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length); extern PyMethodDef _ctypes_module_methods[]; -extern CThunkObject *_ctypes_alloc_callback(PyObject *callable, +extern CThunkObject *_ctypes_alloc_callback(ctypes_state *st, + PyObject *callable, PyObject *converters, PyObject *restype, int flags); @@ -306,7 +316,8 @@ extern int PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info); typedef int(* PPROC)(void); -PyObject *_ctypes_callproc(PPROC pProc, +PyObject *_ctypes_callproc(ctypes_state *st, + PPROC pProc, PyObject *arguments, #ifdef MS_WIN32 IUnknown *pIUnk, @@ -353,14 +364,15 @@ struct tagPyCArgObject { }; #define PyCArg_CheckExact(st, v) Py_IS_TYPE(v, st->PyCArg_Type) -extern PyCArgObject *PyCArgObject_new(void); +extern PyCArgObject *PyCArgObject_new(ctypes_state *st); extern PyObject * -PyCData_get(PyObject *type, GETFUNC getfunc, PyObject *src, +PyCData_get(ctypes_state *st, PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *ptr); extern int -PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +PyCData_set(ctypes_state *st, + PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr); extern void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...); @@ -373,10 +385,7 @@ struct basespec { extern char basespec_string[]; -extern ffi_type *_ctypes_get_ffi_type(PyObject *obj); - -/* exception classes */ -extern PyObject *PyExc_ArgError; +extern ffi_type *_ctypes_get_ffi_type(ctypes_state *st, PyObject *obj); extern char *_ctypes_conversion_encoding; extern char *_ctypes_conversion_errors; @@ -385,16 +394,16 @@ extern char *_ctypes_conversion_errors; extern void _ctypes_free_closure(void *); extern void *_ctypes_alloc_closure(void); -extern PyObject *PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr); +extern PyObject *PyCData_FromBaseObj(ctypes_state *st, PyObject *type, + PyObject *base, Py_ssize_t index, char *adr); extern char *_ctypes_alloc_format_string(const char *prefix, const char *suffix); extern char *_ctypes_alloc_format_string_with_shape(int ndim, const Py_ssize_t *shape, const char *prefix, const char *suffix); -extern int _ctypes_simple_instance(PyObject *obj); +extern int _ctypes_simple_instance(ctypes_state *st, PyObject *obj); -extern PyObject *_ctypes_ptrtype_cache; -PyObject *_ctypes_get_errobj(int **pspace); +PyObject *_ctypes_get_errobj(ctypes_state *st, int **pspace); #ifdef USING_MALLOC_CLOSURE_DOT_C void Py_ffi_closure_free(void *p); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 8666ded5c2b3f2..53e7dc39614d21 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -488,7 +488,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ - prop = PyCField_FromDesc(desc, i, + prop = PyCField_FromDesc(st, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian); @@ -542,7 +542,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct size = 0; offset = 0; align = 0; - prop = PyCField_FromDesc(desc, i, + prop = PyCField_FromDesc(st, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian); diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 92fab9b3998636..65f94e50e1bd7d 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -416,14 +416,14 @@ Modules/xxmodule.c - ErrorObject - ## manually cached PyUnicodeOjbect Modules/_ctypes/callproc.c _ctypes_get_errobj error_object_name - -Modules/_ctypes/_ctypes.c CreateSwappedType suffix - +Modules/_ctypes/_ctypes.c CreateSwappedType swapped_suffix - ##----------------------- ## other ## initialized once Modules/_ctypes/_ctypes.c - _unpickle - -Modules/_ctypes/_ctypes.c PyCArrayType_from_ctype cache - +Modules/_ctypes/_ctypes.c PyCArrayType_from_ctype array_cache - Modules/_cursesmodule.c - ModDict - Modules/_datetimemodule.c datetime_strptime module - From 35b6c4a4da201a947b2ceb96ae4c0d83d4d2df4f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 29 Mar 2024 11:25:17 +0100 Subject: [PATCH 011/143] gh-117347: Fix test_clinic side effects (#117363) Save/restore converters in ClinicWholeFileTest and ClinicExternalTest. --- Lib/test/test_clinic.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 52cb4d6e187855..f95bf858100be6 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -52,6 +52,20 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, return cm.exception +def restore_dict(converters, old_converters): + converters.clear() + converters.update(old_converters) + + +def save_restore_converters(testcase): + testcase.addCleanup(restore_dict, clinic.converters, + clinic.converters.copy()) + testcase.addCleanup(restore_dict, clinic.legacy_converters, + clinic.legacy_converters.copy()) + testcase.addCleanup(restore_dict, clinic.return_converters, + clinic.return_converters.copy()) + + class ClinicWholeFileTest(TestCase): maxDiff = None @@ -60,6 +74,7 @@ def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): filename=filename, lineno=lineno) def setUp(self): + save_restore_converters(self) self.clinic = _make_clinic(filename="test.c") def test_eol(self): @@ -2431,6 +2446,9 @@ def test_state_func_docstring_only_one_param_template(self): class ClinicExternalTest(TestCase): maxDiff = None + def setUp(self): + save_restore_converters(self) + def run_clinic(self, *args): with ( support.captured_stdout() as out, From d9cfe7e565a6e2dc15747a904736264e31a10be4 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 29 Mar 2024 14:14:25 +0300 Subject: [PATCH 012/143] gh-117166: Ignore empty and temporary dirs in `test_makefile` (#117190) --- Lib/test/test_tools/test_makefile.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 7222a054dcd61c..17a1a6d0d38d7d 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -41,9 +41,17 @@ def test_makefile_test_folders(self): self.assertIn(idle_test, test_dirs) used = [idle_test] - for dirpath, _, _ in os.walk(support.TEST_HOME_DIR): + for dirpath, dirs, files in os.walk(support.TEST_HOME_DIR): dirname = os.path.basename(dirpath) - if dirname == '__pycache__': + # Skip temporary dirs: + if dirname == '__pycache__' or dirname.startswith('.'): + dirs.clear() # do not process subfolders + continue + # Skip empty dirs: + if not dirs and not files: + continue + # Skip dirs with hidden-only files: + if files and all(filename.startswith('.') for filename in files): continue relpath = os.path.relpath(dirpath, support.STDLIB_DIR) From 54f7e14500471d1c46fb553adb3ca24cd1fef084 Mon Sep 17 00:00:00 2001 From: Pedro Lacerda Date: Fri, 29 Mar 2024 12:05:00 -0300 Subject: [PATCH 013/143] gh-66449: configparser: Add support for unnamed sections (#117273) Co-authored-by: Jason R. Coombs --- Doc/library/configparser.rst | 31 +++++ Doc/whatsnew/3.13.rst | 6 + Lib/configparser.py | 116 +++++++++++++----- Lib/test/test_configparser.py | 48 ++++++++ ...4-03-28-17-55-22.gh-issue-66449.4jhuEV.rst | 2 + 5 files changed, 172 insertions(+), 31 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-28-17-55-22.gh-issue-66449.4jhuEV.rst diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 445626c267fb6f..9e7638d087a7ce 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -274,6 +274,11 @@ may be treated as parts of multiline values or ignored. By default, a valid section name can be any string that does not contain '\\n'. To change this, see :attr:`ConfigParser.SECTCRE`. +The first section name may be omitted if the parser is configured to allow an +unnamed top level section with ``allow_unnamed_section=True``. In this case, +the keys/values may be retrieved by :const:`UNNAMED_SECTION` as in +``config[UNNAMED_SECTION]``. + Configuration files may include comments, prefixed by specific characters (``#`` and ``;`` by default [1]_). Comments may appear on their own on an otherwise empty line, possibly indented. [1]_ @@ -325,6 +330,27 @@ For example: # Did I mention we can indent comments, too? +.. _unnamed-sections: + +Unnamed Sections +---------------- + +The name of the first section (or unique) may be omitted and values +retrieved by the :const:`UNNAMED_SECTION` attribute. + +.. doctest:: + + >>> config = """ + ... option = value + ... + ... [ Section 2 ] + ... another = val + ... """ + >>> unnamed = configparser.ConfigParser(allow_unnamed_section=True) + >>> unnamed.read_string(config) + >>> unnamed.get(configparser.UNNAMED_SECTION, 'option') + 'value' + Interpolation of values ----------------------- @@ -1216,6 +1242,11 @@ ConfigParser Objects names is stripped before :meth:`optionxform` is called. +.. data:: UNNAMED_SECTION + + A special object representing a section name used to reference the unnamed section (see :ref:`unnamed-sections`). + + .. data:: MAX_INTERPOLATION_DEPTH The maximum depth for recursive interpolation for :meth:`~configparser.ConfigParser.get` when the *raw* diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 5a5c506d83d735..f50364a7ddcc2a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -214,6 +214,12 @@ Other Language Changes (Contributed by William Woodruff in :gh:`112389`.) +* The :class:`configparser.ConfigParser` now accepts unnamed sections before named + ones if configured to do so. + + (Contributed by Pedro Sousa Lacerda in :gh:`66449`) + + New Modules =========== diff --git a/Lib/configparser.py b/Lib/configparser.py index 8f182eec306b8b..3040e1fbe5b9c1 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -18,8 +18,8 @@ delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', - interpolation=, converters=): - + interpolation=, converters=, + allow_unnamed_section=False): Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. @@ -68,6 +68,10 @@ converter gets its corresponding get*() method on the parser object and section proxies. + When `allow_unnamed_section` is True (default: False), options + without section are accepted: the section for these is + ``configparser.UNNAMED_SECTION``. + sections() Return all the configuration section names, sans DEFAULT. @@ -156,7 +160,7 @@ "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "SectionProxy", "ConverterMapping", - "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH") + "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") _default_dict = dict DEFAULTSECT = "DEFAULT" @@ -336,6 +340,15 @@ def __init__(self, filename, lineno, line): self.line = line self.args = (filename, lineno, line) +class _UnnamedSection: + + def __repr__(self): + return "" + + +UNNAMED_SECTION = _UnnamedSection() + + # Used in parser getters to indicate the default behaviour when a specific # option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. @@ -550,7 +563,8 @@ def __init__(self, defaults=None, dict_type=_default_dict, comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, - interpolation=_UNSET, converters=_UNSET): + interpolation=_UNSET, converters=_UNSET, + allow_unnamed_section=False,): self._dict = dict_type self._sections = self._dict() @@ -589,6 +603,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, self._converters.update(converters) if defaults: self._read_defaults(defaults) + self._allow_unnamed_section = allow_unnamed_section def defaults(self): return self._defaults @@ -862,13 +877,19 @@ def write(self, fp, space_around_delimiters=True): if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) + if UNNAMED_SECTION in self._sections: + self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True) + for section in self._sections: + if section is UNNAMED_SECTION: + continue self._write_section(fp, section, self._sections[section].items(), d) - def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp`.""" - fp.write("[{}]\n".format(section_name)) + def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): + """Write a single section to the specified `fp'.""" + if not unnamed: + fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, value) @@ -961,6 +982,7 @@ def _read(self, fp, fpname): lineno = 0 indent_level = 0 e = None # None, or an exception + try: for lineno, line in enumerate(fp, start=1): comment_start = sys.maxsize @@ -1007,6 +1029,13 @@ def _read(self, fp, fpname): cursect[optname].append(value) # a section header or option header? else: + if self._allow_unnamed_section and cursect is None: + sectname = UNNAMED_SECTION + cursect = self._dict() + self._sections[sectname] = cursect + self._proxies[sectname] = SectionProxy(self, sectname) + elements_added.add(sectname) + indent_level = cur_indent_level # is it a section header? mo = self.SECTCRE.match(value) @@ -1027,36 +1056,61 @@ def _read(self, fp, fpname): elements_added.add(sectname) # So sections can't start with a continuation line optname = None - # no section header in the file? + # no section header? elif cursect is None: raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? + # an option line? else: - mo = self._optcre.match(value) + indent_level = cur_indent_level + # is it a section header? + mo = self.SECTCRE.match(value) if mo: - optname, vi, optval = mo.group('option', 'vi', 'value') - if not optname: - e = self._handle_error(e, fpname, lineno, line) - optname = self.optionxform(optname.rstrip()) - if (self._strict and - (sectname, optname) in elements_added): - raise DuplicateOptionError(sectname, optname, - fpname, lineno) - elements_added.add((sectname, optname)) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - optval = optval.strip() - cursect[optname] = [optval] + sectname = mo.group('header') + if sectname in self._sections: + if self._strict and sectname in elements_added: + raise DuplicateSectionError(sectname, fpname, + lineno) + cursect = self._sections[sectname] + elements_added.add(sectname) + elif sectname == self.default_section: + cursect = self._defaults else: - # valueless option handling - cursect[optname] = None + cursect = self._dict() + self._sections[sectname] = cursect + self._proxies[sectname] = SectionProxy(self, sectname) + elements_added.add(sectname) + # So sections can't start with a continuation line + optname = None + # no section header in the file? + elif cursect is None: + raise MissingSectionHeaderError(fpname, lineno, line) + # an option line? else: - # a non-fatal parsing error occurred. set up the - # exception but keep going. the exception will be - # raised at the end of the file and will contain a - # list of all bogus lines - e = self._handle_error(e, fpname, lineno, line) + mo = self._optcre.match(value) + if mo: + optname, vi, optval = mo.group('option', 'vi', 'value') + if not optname: + e = self._handle_error(e, fpname, lineno, line) + optname = self.optionxform(optname.rstrip()) + if (self._strict and + (sectname, optname) in elements_added): + raise DuplicateOptionError(sectname, optname, + fpname, lineno) + elements_added.add((sectname, optname)) + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + optval = optval.strip() + cursect[optname] = [optval] + else: + # valueless option handling + cursect[optname] = None + else: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + e = self._handle_error(e, fpname, lineno, line) finally: self._join_multiline_values() # if any parsing errors occurred, raise an exception diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 6340e378c4f21a..fe09472db89cd2 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2115,6 +2115,54 @@ def test_instance_assignment(self): self.assertEqual(cfg['two'].getlen('one'), 5) +class SectionlessTestCase(unittest.TestCase): + + def fromstring(self, string): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.read_string(string) + return cfg + + def test_no_first_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + [sect1] + c = 3 + """) + + self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg1.sections())) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg1['sect1']['c']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + #self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg2.sections())) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg2['sect1']['c']) + + def test_no_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + """) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg1.sections()) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg2.sections()) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, configparser, not_exported={"Error"}) diff --git a/Misc/NEWS.d/next/Library/2024-03-28-17-55-22.gh-issue-66449.4jhuEV.rst b/Misc/NEWS.d/next/Library/2024-03-28-17-55-22.gh-issue-66449.4jhuEV.rst new file mode 100644 index 00000000000000..898100b87e1dbd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-28-17-55-22.gh-issue-66449.4jhuEV.rst @@ -0,0 +1,2 @@ +:class:`configparser.ConfigParser` now accepts unnamed sections before named +ones, if configured to do so. From 0fa571dbcdf19b541276cb00bb929381930467b2 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 29 Mar 2024 09:02:01 -0700 Subject: [PATCH 014/143] Refactor pdb executable targets (#112570) Co-authored-by: Jason R. Coombs --- Lib/pdb.py | 77 +++++++++++++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index f80171d172b23e..d4138b95d3c332 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -82,13 +82,12 @@ import signal import inspect import tokenize -import functools import traceback import linecache from contextlib import contextmanager from rlcompleter import Completer -from typing import Union +from types import CodeType class Restart(Exception): @@ -156,52 +155,58 @@ def __repr__(self): return self -class _ScriptTarget(str): - def __new__(cls, val): - # Mutate self to be the "real path". - res = super().__new__(cls, os.path.realpath(val)) +class _ExecutableTarget: + filename: str + code: CodeType | str + namespace: dict - # Store the original path for error reporting. - res.orig = val - return res +class _ScriptTarget(_ExecutableTarget): + def __init__(self, target): + self._target = os.path.realpath(target) - def check(self): - if not os.path.exists(self): - print('Error:', self.orig, 'does not exist') + if not os.path.exists(self._target): + print(f'Error: {target} does not exist') sys.exit(1) - if os.path.isdir(self): - print('Error:', self.orig, 'is a directory') + if os.path.isdir(self._target): + print(f'Error: {target} is a directory') sys.exit(1) # If safe_path(-P) is not set, sys.path[0] is the directory # of pdb, and we should replace it with the directory of the script if not sys.flags.safe_path: - sys.path[0] = os.path.dirname(self) + sys.path[0] = os.path.dirname(self._target) + + def __repr__(self): + return self._target @property def filename(self): - return self + return self._target + + @property + def code(self): + # Open the file each time because the file may be modified + with io.open_code(self._target) as fp: + return f"exec(compile({fp.read()!r}, {self._target!r}, 'exec'))" @property def namespace(self): return dict( __name__='__main__', - __file__=self, + __file__=self._target, __builtins__=__builtins__, __spec__=None, ) - @property - def code(self): - with io.open_code(self) as fp: - return f"exec(compile({fp.read()!r}, {self!r}, 'exec'))" +class _ModuleTarget(_ExecutableTarget): + def __init__(self, target): + self._target = target -class _ModuleTarget(str): - def check(self): + import runpy try: - self._details + _, self._spec, self._code = runpy._get_module_details(self._target) except ImportError as e: print(f"ImportError: {e}") sys.exit(1) @@ -209,24 +214,16 @@ def check(self): traceback.print_exc() sys.exit(1) - @functools.cached_property - def _details(self): - import runpy - return runpy._get_module_details(self) + def __repr__(self): + return self._target @property def filename(self): - return self.code.co_filename + return self._code.co_filename @property def code(self): - name, spec, code = self._details - return code - - @property - def _spec(self): - name, spec, code = self._details - return spec + return self._code @property def namespace(self): @@ -2029,7 +2026,7 @@ def lookupmodule(self, filename): return fullname return None - def _run(self, target: Union[_ModuleTarget, _ScriptTarget]): + def _run(self, target: _ExecutableTarget): # When bdb sets tracing, a number of call and line events happen # BEFORE debugger even reaches user's code (and the exact sequence of # events depends on python version). Take special measures to @@ -2281,8 +2278,6 @@ def main(): file = opts.args.pop(0) target = _ScriptTarget(file) - target.check() - sys.argv[:] = [file] + opts.args # Hide "pdb.py" and pdb options from argument list # Note on saving/restoring sys.argv: it's a good idea when sys.argv was @@ -2306,8 +2301,8 @@ def main(): print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") pdb.interaction(None, e) - print("Post mortem debugger finished. The " + target + - " will be restarted") + print(f"Post mortem debugger finished. The {target} will " + "be restarted") if pdb._user_requested_quit: break print("The program finished and will be restarted") From ddf95b5f16031cdbd0d728e55eb06dff002a8678 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 29 Mar 2024 18:26:06 +0100 Subject: [PATCH 015/143] gh-116664: Fix unused var warnings in _warnings.c in non-free-threaded builds (#117373) The warnings were introduced by commit c1712ef06. --- Python/_warnings.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Python/_warnings.c b/Python/_warnings.c index 66a460e2a2c509..ac3d3cc2d1246b 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -994,8 +994,10 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level, &filename, &lineno, &module, ®istry)) return NULL; +#ifdef Py_GIL_DISABLED WarningsState *st = warnings_get_state(tstate->interp); assert(st != NULL); +#endif Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, @@ -1149,8 +1151,10 @@ warnings_warn_explicit_impl(PyObject *module, PyObject *message, } } +#ifdef Py_GIL_DISABLED WarningsState *st = warnings_get_state(tstate->interp); assert(st != NULL); +#endif Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); returned = warn_explicit(tstate, category, message, filename, lineno, @@ -1296,8 +1300,10 @@ PyErr_WarnExplicitObject(PyObject *category, PyObject *message, return -1; } +#ifdef Py_GIL_DISABLED WarningsState *st = warnings_get_state(tstate->interp); assert(st != NULL); +#endif Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); res = warn_explicit(tstate, category, message, filename, lineno, @@ -1367,8 +1373,10 @@ PyErr_WarnExplicitFormat(PyObject *category, PyObject *res; PyThreadState *tstate = get_current_tstate(); if (tstate != NULL) { +#ifdef Py_GIL_DISABLED WarningsState *st = warnings_get_state(tstate->interp); assert(st != NULL); +#endif Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); res = warn_explicit(tstate, category, message, filename, lineno, From f05fb2e65c2dffdfae940f2707765c4994925205 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 29 Mar 2024 13:33:04 -0400 Subject: [PATCH 016/143] gh-112529: Don't untrack tuples or dicts with zero refcount (#117370) The free-threaded GC sometimes sees objects with zero refcount. This can happen due to the delay in merging biased reference counting fields, and, in the future, due to deferred reference counting. We should not untrack these objects or they will never be collected. This fixes the refleaks in the free-threaded build. --- Python/gc_free_threading.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 69ce22a1e83b62..4524382e4f689f 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -374,25 +374,28 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } - // Untrack tuples and dicts as necessary in this pass. - if (PyTuple_CheckExact(op)) { - _PyTuple_MaybeUntrack(op); - if (!_PyObject_GC_IS_TRACKED(op)) { - gc_restore_refs(op); - return true; + Py_ssize_t refcount = Py_REFCNT(op); + _PyObject_ASSERT(op, refcount >= 0); + + if (refcount > 0) { + // Untrack tuples and dicts as necessary in this pass, but not objects + // with zero refcount, which we will want to collect. + if (PyTuple_CheckExact(op)) { + _PyTuple_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + gc_restore_refs(op); + return true; + } } - } - else if (PyDict_CheckExact(op)) { - _PyDict_MaybeUntrack(op); - if (!_PyObject_GC_IS_TRACKED(op)) { - gc_restore_refs(op); - return true; + else if (PyDict_CheckExact(op)) { + _PyDict_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + gc_restore_refs(op); + return true; + } } } - Py_ssize_t refcount = Py_REFCNT(op); - _PyObject_ASSERT(op, refcount >= 0); - // We repurpose ob_tid to compute "gc_refs", the number of external // references to the object (i.e., from outside the GC heaps). This means // that ob_tid is no longer a valid thread id until it is restored by From 397d88db5e9ab2a43de3fdf5f8b973a949edc405 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 29 Mar 2024 13:34:04 -0400 Subject: [PATCH 017/143] gh-117344: Skip flaky tests in free-threaded build (#117355) The tests are not reliable with the GIL disabled. In theory, they can fail with the GIL enabled too, but the failures are much more likely with the GIL disabled. --- Lib/test/test_concurrent_futures/test_process_pool.py | 1 + Lib/test/test_concurrent_futures/test_thread_pool.py | 1 + 2 files changed, 2 insertions(+) diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 70444bb147fadc..e60e7a6607a997 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -116,6 +116,7 @@ def test_saturation(self): for _ in range(job_count): sem.release() + @unittest.skipIf(support.Py_GIL_DISABLED, "gh-117344: test is flaky without the GIL") def test_idle_process_reuse_one(self): executor = self.executor assert executor._max_workers >= 4 diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 16043fd1235614..86e65265516c3f 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -41,6 +41,7 @@ def acquire_lock(lock): sem.release() executor.shutdown(wait=True) + @unittest.skipIf(support.Py_GIL_DISABLED, "gh-117344: test is flaky without the GIL") def test_idle_thread_reuse(self): executor = self.executor_type() executor.submit(mul, 21, 2).result() From 19c1dd60c5b53fb0533610ad139ef591294f26e8 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 29 Mar 2024 13:35:43 -0400 Subject: [PATCH 018/143] gh-117323: Make `cell` thread-safe in free-threaded builds (#117330) Use critical sections to lock around accesses to cell contents. The critical sections are no-ops in the default (with GIL) build. --- Include/internal/pycore_cell.h | 48 ++++++++++++++++++++++++++++++ Makefile.pre.in | 1 + Objects/cellobject.c | 8 ++--- PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 ++ Python/bytecodes.c | 20 +++++-------- Python/ceval.c | 1 + Python/executor_cases.c.h | 19 +++++------- Python/generated_cases.c.h | 19 +++++------- Tools/cases_generator/analyzer.py | 5 ++-- Tools/jit/template.c | 1 + 11 files changed, 83 insertions(+), 43 deletions(-) create mode 100644 Include/internal/pycore_cell.h diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h new file mode 100644 index 00000000000000..27f67d57b2fb79 --- /dev/null +++ b/Include/internal/pycore_cell.h @@ -0,0 +1,48 @@ +#ifndef Py_INTERNAL_CELL_H +#define Py_INTERNAL_CELL_H + +#include "pycore_critical_section.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Sets the cell contents to `value` and return previous contents. Steals a +// reference to `value`. +static inline PyObject * +PyCell_SwapTakeRef(PyCellObject *cell, PyObject *value) +{ + PyObject *old_value; + Py_BEGIN_CRITICAL_SECTION(cell); + old_value = cell->ob_ref; + cell->ob_ref = value; + Py_END_CRITICAL_SECTION(); + return old_value; +} + +static inline void +PyCell_SetTakeRef(PyCellObject *cell, PyObject *value) +{ + PyObject *old_value = PyCell_SwapTakeRef(cell, value); + Py_XDECREF(old_value); +} + +// Gets the cell contents. Returns a new reference. +static inline PyObject * +PyCell_GetRef(PyCellObject *cell) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(cell); + res = Py_XNewRef(cell->ob_ref); + Py_END_CRITICAL_SECTION(); + return res; +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CELL_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 5b89d6ba1acf71..f5c2af0696ac33 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1130,6 +1130,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_bytesobject.h \ $(srcdir)/Include/internal/pycore_call.h \ $(srcdir)/Include/internal/pycore_capsule.h \ + $(srcdir)/Include/internal/pycore_cell.h \ $(srcdir)/Include/internal/pycore_ceval.h \ $(srcdir)/Include/internal/pycore_ceval_state.h \ $(srcdir)/Include/internal/pycore_code.h \ diff --git a/Objects/cellobject.c b/Objects/cellobject.c index f1a43be38b2b58..b1154e4ca4ace6 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -1,6 +1,7 @@ /* Cell object implementation */ #include "Python.h" +#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" @@ -56,8 +57,7 @@ PyCell_Get(PyObject *op) PyErr_BadInternalCall(); return NULL; } - PyObject *value = PyCell_GET(op); - return Py_XNewRef(value); + return PyCell_GetRef((PyCellObject *)op); } int @@ -67,9 +67,7 @@ PyCell_Set(PyObject *op, PyObject *value) PyErr_BadInternalCall(); return -1; } - PyObject *old_value = PyCell_GET(op); - PyCell_SET(op, Py_XNewRef(value)); - Py_XDECREF(old_value); + PyCell_SetTakeRef((PyCellObject *)op, Py_XNewRef(value)); return 0; } diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c944bbafdba7e5..7a2a98df6511a1 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -210,6 +210,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0afad125ce1e97..89b56ec1267104 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -555,6 +555,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5cd9db97c71e37..bfb378c4a41500 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -8,6 +8,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" @@ -1523,14 +1524,13 @@ dummy_func( inst(DELETE_DEREF, (--)) { PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_NO_POP(); } - PyCell_SET(cell, NULL); Py_DECREF(oldobj); } @@ -1543,32 +1543,28 @@ dummy_func( ERROR_NO_POP(); } if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = PyCell_GetRef(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_NO_POP(); } - Py_INCREF(value); } Py_DECREF(class_dict); } inst(LOAD_DEREF, ( -- value)) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = PyCell_GetRef(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_IF(true, error); } - Py_INCREF(value); } inst(STORE_DEREF, (v --)) { - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + PyCell_SetTakeRef(cell, v); } inst(COPY_FREE_VARS, (--)) { diff --git a/Python/ceval.c b/Python/ceval.c index d34db61eecbae2..1b13eb1702355f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5,6 +5,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_ceval.h" #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 224b600b8f6a4a..ce0dc235c54fcf 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1362,14 +1362,13 @@ case _DELETE_DEREF: { oparg = CURRENT_OPARG(); PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); JUMP_TO_ERROR(); } - PyCell_SET(cell, NULL); Py_DECREF(oldobj); break; } @@ -1387,13 +1386,12 @@ JUMP_TO_ERROR(); } if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = PyCell_GetRef(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); JUMP_TO_ERROR(); } - Py_INCREF(value); } Py_DECREF(class_dict); stack_pointer[-1] = value; @@ -1403,13 +1401,12 @@ case _LOAD_DEREF: { PyObject *value; oparg = CURRENT_OPARG(); - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = PyCell_GetRef(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); if (true) JUMP_TO_ERROR(); } - Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; break; @@ -1419,10 +1416,8 @@ PyObject *v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + PyCell_SetTakeRef(cell, v); stack_pointer += -1; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c66eb678d38475..e8e2397b11cd48 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2320,14 +2320,13 @@ next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } - PyCell_SET(cell, NULL); Py_DECREF(oldobj); DISPATCH(); } @@ -4096,13 +4095,12 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); PyObject *value; - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = PyCell_GetRef(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); if (true) goto error; } - Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); @@ -4186,13 +4184,12 @@ goto error; } if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + value = PyCell_GetRef(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } - Py_INCREF(value); } Py_DECREF(class_dict); stack_pointer[-1] = value; @@ -5436,10 +5433,8 @@ INSTRUCTION_STATS(STORE_DEREF); PyObject *v; v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); + PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + PyCell_SetTakeRef(cell, v); stack_pointer += -1; DISPATCH(); } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 2329205ad31d09..ddafcf99ca1e37 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -520,8 +520,9 @@ def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: def compute_properties(op: parser.InstDef) -> Properties: has_free = ( variable_used(op, "PyCell_New") - or variable_used(op, "PyCell_GET") - or variable_used(op, "PyCell_SET") + or variable_used(op, "PyCell_GetRef") + or variable_used(op, "PyCell_SetTakeRef") + or variable_used(op, "PyCell_SwapTakeRef") ) deopts_if = variable_used(op, "DEOPT_IF") exits_if = variable_used(op, "EXIT_IF") diff --git a/Tools/jit/template.c b/Tools/jit/template.c index f8be4d7f78facd..54160084cda460 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -2,6 +2,7 @@ #include "pycore_call.h" #include "pycore_ceval.h" +#include "pycore_cell.h" #include "pycore_dict.h" #include "pycore_emscripten_signal.h" #include "pycore_intrinsics.h" From 5d21d884b6ffa45dac50a5f9a07c41356a8478b4 Mon Sep 17 00:00:00 2001 From: mpage Date: Fri, 29 Mar 2024 10:42:02 -0700 Subject: [PATCH 019/143] gh-111926: Avoid locking in PyType_IsSubtype (#117275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read the MRO in a thread-unsafe way in `PyType_IsSubtype` to avoid locking. Fixing this is tracked in #117306. The motivation for this change is in support of making weakrefs thread-safe in free-threaded builds: `WeakValueDictionary` uses a special dictionary function, `_PyDict_DelItemIf` to remove dead weakrefs from the dictionary. `_PyDict_DelItemIf` removes a key if a user supplied predicate evaluates to true for the value associated with the key. Crucially for the `WeakValueDictionary` use case, the predicate evaluation + deletion sequence is atomic, provided that the predicate doesn’t suspend. The predicate used by `WeakValueDictionary` includes a subtype check, which we must ensure doesn't suspend in free-threaded builds. --- Objects/typeobject.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 82822784aaf407..2ef79fbf17b329 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2341,14 +2341,7 @@ is_subtype_with_mro(PyObject *a_mro, PyTypeObject *a, PyTypeObject *b) int PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) { -#ifdef Py_GIL_DISABLED - PyObject *mro = _PyType_GetMRO(a); - int res = is_subtype_with_mro(mro, a, b); - Py_XDECREF(mro); - return res; -#else - return is_subtype_with_mro(lookup_tp_mro(a), a, b); -#endif + return is_subtype_with_mro(a->tp_mro, a, b); } /* Routines to do a method lookup in the type without looking in the From 94c97423a9c4969f8ddd4a3aa4aacb99c4d5263d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 Mar 2024 11:31:09 -0700 Subject: [PATCH 020/143] Fix broken format in error for bad input in summarize_stats.py (#117375) When you pass the script a non-existent input file, you get a TypeError instead of the intended ValueError. --- Tools/scripts/summarize_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index d40106b8682388..8dc590b4b89a88 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -114,7 +114,7 @@ def load_raw_data(input: Path) -> RawData: return data else: - raise ValueError(f"{input:r} is not a file or directory path") + raise ValueError(f"{input} is not a file or directory path") def save_raw_data(data: RawData, json_output: TextIO): From 01bd74eadbc4ff839d39762fae6366f50c1e116e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 29 Mar 2024 15:33:06 -0400 Subject: [PATCH 021/143] gh-117300: Use stop the world to make `sys._current_frames` and `sys._current_exceptions` thread-safe. (#117301) This adds a stop the world pause to make the two functions thread-safe when the GIL is disabled in the free-threaded build. Additionally, the main test thread may call `sys._current_exceptions()` as soon as `g_raised.set()` is called. The background thread may not yet reach the `leave_g.wait()` line. --- Lib/test/test_sys.py | 3 ++- Python/pystate.c | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 37c16cd1047885..f6f23b0afc34c6 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -562,7 +562,8 @@ def g456(): # And the next record must be for g456(). filename, lineno, funcname, sourceline = stack[i+1] self.assertEqual(funcname, "g456") - self.assertTrue(sourceline.startswith("if leave_g.wait(")) + self.assertTrue((sourceline.startswith("if leave_g.wait(") or + sourceline.startswith("g_raised.set()"))) finally: # Reap the spawned thread. leave_g.set() diff --git a/Python/pystate.c b/Python/pystate.c index 8489f53c6e3e34..8bec72779b2c24 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2408,6 +2408,7 @@ _PyThread_CurrentFrames(void) * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ + _PyEval_StopTheWorldAll(runtime); HEAD_LOCK(runtime); PyInterpreterState *i; for (i = runtime->interpreters.head; i != NULL; i = i->next) { @@ -2441,6 +2442,7 @@ _PyThread_CurrentFrames(void) done: HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(runtime); return result; } @@ -2472,6 +2474,7 @@ _PyThread_CurrentExceptions(void) * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ + _PyEval_StopTheWorldAll(runtime); HEAD_LOCK(runtime); PyInterpreterState *i; for (i = runtime->interpreters.head; i != NULL; i = i->next) { @@ -2504,6 +2507,7 @@ _PyThread_CurrentExceptions(void) done: HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(runtime); return result; } From 019143fecbfc26e69800d28d2a9e3392a051780b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 29 Mar 2024 16:06:09 -0400 Subject: [PATCH 022/143] gh-117348: Refactored RawConfigParser._read for similicity and comprehensibility (#117372) * Extract method for _read_inner, reducing complexity and indentation by 1. * Extract method for _raise_all and yield ParseErrors from _read_inner. Reduces complexity by 1 and reduces touch points for handling errors in _read_inner. * Prefer iterators to splat expansion and literal indexing. * Extract method for _strip_comments. Reduces complexity by 7. * Model the file lines in a class to encapsulate the comment status and cleaned value. * Encapsulate the read state as a dataclass * Extract _handle_continuation_line and _handle_rest methods. Reduces complexity by 8. * Reindent * At least for now, collect errors in the ReadState * Check for missing section header separately. * Extract methods for _handle_header and _handle_option. Reduces complexity by 6. * Remove unreachable code. Reduces complexity by 4. * Remove unreachable branch * Handle error condition early. Reduces complexity by 1. * Add blurb * Move _raise_all to ParsingError, as its behavior is most closely related to the exception class and not the reader. * Split _strip* into separate methods. * Refactor _strip_full to compute the strip just once and use 'not any' to determine the factor. * Replace use of 'sys.maxsize' with direct computation of the stripped value. * Extract has_comments as a dynamic property. * Implement clean as a cached property. * Model comment prefixes in the RawConfigParser within a prefixes namespace. * Use a regular expression to search for the first match. Avoids mutating variables and tricky logic and over-computing all of the starts when only the first is relevant. --- Lib/configparser.py | 330 ++++++++++-------- ...-03-29-12-07-26.gh-issue-117348.WjCYvK.rst | 2 + 2 files changed, 185 insertions(+), 147 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-29-12-07-26.gh-issue-117348.WjCYvK.rst diff --git a/Lib/configparser.py b/Lib/configparser.py index 3040e1fbe5b9c1..d0326c60e9b907 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -145,12 +145,15 @@ from collections.abc import MutableMapping from collections import ChainMap as _ChainMap +import contextlib +from dataclasses import dataclass, field import functools import io import itertools import os import re import sys +from typing import Iterable __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", @@ -302,15 +305,33 @@ def __init__(self, option, section, rawval): class ParsingError(Error): """Raised when a configuration file does not follow legal syntax.""" - def __init__(self, source): + def __init__(self, source, *args): super().__init__(f'Source contains parsing errors: {source!r}') self.source = source self.errors = [] self.args = (source, ) + if args: + self.append(*args) def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, line) + self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + + def combine(self, others): + for other in others: + for error in other.errors: + self.append(*error) + return self + + @staticmethod + def _raise_all(exceptions: Iterable['ParsingError']): + """ + Combine any number of ParsingErrors into one and raise it. + """ + exceptions = iter(exceptions) + with contextlib.suppress(StopIteration): + raise next(exceptions).combine(exceptions) + class MissingSectionHeaderError(ParsingError): @@ -517,6 +538,55 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, "found: %r" % (rest,)) +@dataclass +class _ReadState: + elements_added : set[str] = field(default_factory=set) + cursect : dict[str, str] | None = None + sectname : str | None = None + optname : str | None = None + lineno : int = 0 + indent_level : int = 0 + errors : list[ParsingError] = field(default_factory=list) + + +@dataclass +class _Prefixes: + full : Iterable[str] + inline : Iterable[str] + + +class _Line(str): + + def __new__(cls, val, *args, **kwargs): + return super().__new__(cls, val) + + def __init__(self, val, prefixes: _Prefixes): + self.prefixes = prefixes + + @functools.cached_property + def clean(self): + return self._strip_full() and self._strip_inline() + + @property + def has_comments(self): + return self.strip() != self.clean + + def _strip_inline(self): + """ + Search for the earliest prefix at the beginning of the line or following a space. + """ + matcher = re.compile( + '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) + # match nothing if no prefixes + or '(?!)' + ) + match = matcher.search(self) + return self[:match.start() if match else None].strip() + + def _strip_full(self): + return '' if any(map(self.strip().startswith, self.prefixes.full)) else True + + class RawConfigParser(MutableMapping): """ConfigParser that does not do interpolation.""" @@ -583,8 +653,10 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._comment_prefixes = tuple(comment_prefixes or ()) - self._inline_comment_prefixes = tuple(inline_comment_prefixes or ()) + self._prefixes = _Prefixes( + full=tuple(comment_prefixes or ()), + inline=tuple(inline_comment_prefixes or ()), + ) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values @@ -975,147 +1047,117 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - elements_added = set() - cursect = None # None, or a dictionary - sectname = None - optname = None - lineno = 0 - indent_level = 0 - e = None # None, or an exception try: - for lineno, line in enumerate(fp, start=1): - comment_start = sys.maxsize - # strip inline comments - inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} - while comment_start == sys.maxsize and inline_prefixes: - next_prefixes = {} - for prefix, index in inline_prefixes.items(): - index = line.find(prefix, index+1) - if index == -1: - continue - next_prefixes[prefix] = index - if index == 0 or (index > 0 and line[index-1].isspace()): - comment_start = min(comment_start, index) - inline_prefixes = next_prefixes - # strip full line comments - for prefix in self._comment_prefixes: - if line.strip().startswith(prefix): - comment_start = 0 - break - if comment_start == sys.maxsize: - comment_start = None - value = line[:comment_start].strip() - if not value: - if self._empty_lines_in_values: - # add empty line to the value, but only if there was no - # comment on the line - if (comment_start is None and - cursect is not None and - optname and - cursect[optname] is not None): - cursect[optname].append('') # newlines added at join - else: - # empty line marks end of value - indent_level = sys.maxsize - continue - # continuation line? - first_nonspace = self.NONSPACECRE.search(line) - cur_indent_level = first_nonspace.start() if first_nonspace else 0 - if (cursect is not None and optname and - cur_indent_level > indent_level): - if cursect[optname] is None: - raise MultilineContinuationError(fpname, lineno, line) - cursect[optname].append(value) - # a section header or option header? - else: - if self._allow_unnamed_section and cursect is None: - sectname = UNNAMED_SECTION - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - - indent_level = cur_indent_level - # is it a section header? - mo = self.SECTCRE.match(value) - if mo: - sectname = mo.group('header') - if sectname in self._sections: - if self._strict and sectname in elements_added: - raise DuplicateSectionError(sectname, fpname, - lineno) - cursect = self._sections[sectname] - elements_added.add(sectname) - elif sectname == self.default_section: - cursect = self._defaults - else: - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - # So sections can't start with a continuation line - optname = None - # no section header? - elif cursect is None: - raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? - else: - indent_level = cur_indent_level - # is it a section header? - mo = self.SECTCRE.match(value) - if mo: - sectname = mo.group('header') - if sectname in self._sections: - if self._strict and sectname in elements_added: - raise DuplicateSectionError(sectname, fpname, - lineno) - cursect = self._sections[sectname] - elements_added.add(sectname) - elif sectname == self.default_section: - cursect = self._defaults - else: - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - # So sections can't start with a continuation line - optname = None - # no section header in the file? - elif cursect is None: - raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? - else: - mo = self._optcre.match(value) - if mo: - optname, vi, optval = mo.group('option', 'vi', 'value') - if not optname: - e = self._handle_error(e, fpname, lineno, line) - optname = self.optionxform(optname.rstrip()) - if (self._strict and - (sectname, optname) in elements_added): - raise DuplicateOptionError(sectname, optname, - fpname, lineno) - elements_added.add((sectname, optname)) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - optval = optval.strip() - cursect[optname] = [optval] - else: - # valueless option handling - cursect[optname] = None - else: - # a non-fatal parsing error occurred. set up the - # exception but keep going. the exception will be - # raised at the end of the file and will contain a - # list of all bogus lines - e = self._handle_error(e, fpname, lineno, line) + ParsingError._raise_all(self._read_inner(fp, fpname)) finally: self._join_multiline_values() - # if any parsing errors occurred, raise an exception - if e: - raise e + + def _read_inner(self, fp, fpname): + st = _ReadState() + + Line = functools.partial(_Line, prefixes=self._prefixes) + for st.lineno, line in enumerate(map(Line, fp), start=1): + if not line.clean: + if self._empty_lines_in_values: + # add empty line to the value, but only if there was no + # comment on the line + if (not line.has_comments and + st.cursect is not None and + st.optname and + st.cursect[st.optname] is not None): + st.cursect[st.optname].append('') # newlines added at join + else: + # empty line marks end of value + st.indent_level = sys.maxsize + continue + + first_nonspace = self.NONSPACECRE.search(line) + st.cur_indent_level = first_nonspace.start() if first_nonspace else 0 + + if self._handle_continuation_line(st, line, fpname): + continue + + self._handle_rest(st, line, fpname) + + return st.errors + + def _handle_continuation_line(self, st, line, fpname): + # continuation line? + is_continue = (st.cursect is not None and st.optname and + st.cur_indent_level > st.indent_level) + if is_continue: + if st.cursect[st.optname] is None: + raise MultilineContinuationError(fpname, st.lineno, line) + st.cursect[st.optname].append(line.clean) + return is_continue + + def _handle_rest(self, st, line, fpname): + # a section header or option header? + if self._allow_unnamed_section and st.cursect is None: + st.sectname = UNNAMED_SECTION + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + + st.indent_level = st.cur_indent_level + # is it a section header? + mo = self.SECTCRE.match(line.clean) + + if not mo and st.cursect is None: + raise MissingSectionHeaderError(fpname, st.lineno, line) + + self._handle_header(st, mo, fpname) if mo else self._handle_option(st, line, fpname) + + def _handle_header(self, st, mo, fpname): + st.sectname = mo.group('header') + if st.sectname in self._sections: + if self._strict and st.sectname in st.elements_added: + raise DuplicateSectionError(st.sectname, fpname, + st.lineno) + st.cursect = self._sections[st.sectname] + st.elements_added.add(st.sectname) + elif st.sectname == self.default_section: + st.cursect = self._defaults + else: + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + # So sections can't start with a continuation line + st.optname = None + + def _handle_option(self, st, line, fpname): + # an option line? + st.indent_level = st.cur_indent_level + + mo = self._optcre.match(line.clean) + if not mo: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + st.errors.append(ParsingError(fpname, st.lineno, line)) + return + + st.optname, vi, optval = mo.group('option', 'vi', 'value') + if not st.optname: + st.errors.append(ParsingError(fpname, st.lineno, line)) + st.optname = self.optionxform(st.optname.rstrip()) + if (self._strict and + (st.sectname, st.optname) in st.elements_added): + raise DuplicateOptionError(st.sectname, st.optname, + fpname, st.lineno) + st.elements_added.add((st.sectname, st.optname)) + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + optval = optval.strip() + st.cursect[st.optname] = [optval] + else: + # valueless option handling + st.cursect[st.optname] = None def _join_multiline_values(self): defaults = self.default_section, self._defaults @@ -1135,12 +1177,6 @@ def _read_defaults(self, defaults): for key, value in defaults.items(): self._defaults[self.optionxform(key)] = value - def _handle_error(self, exc, fpname, lineno, line): - if not exc: - exc = ParsingError(fpname) - exc.append(lineno, repr(line)) - return exc - def _unify_values(self, section, vars): """Create a sequence of lookups with 'vars' taking priority over the 'section' which takes priority over the DEFAULTSECT. diff --git a/Misc/NEWS.d/next/Library/2024-03-29-12-07-26.gh-issue-117348.WjCYvK.rst b/Misc/NEWS.d/next/Library/2024-03-29-12-07-26.gh-issue-117348.WjCYvK.rst new file mode 100644 index 00000000000000..cd3006c3b7b8f0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-29-12-07-26.gh-issue-117348.WjCYvK.rst @@ -0,0 +1,2 @@ +Refactored :meth:`configparser.RawConfigParser._read` to reduce cyclometric +complexity and improve comprehensibility. From 05e0b67a43c5c1778dc2643c8b7c12864e135999 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 29 Mar 2024 21:23:28 +0100 Subject: [PATCH 023/143] gh-116664: In _warnings.c, make filters_version access thread-safe (#117374) - assert that the lock is held in already_warned() - protect 'filters_version' increment in warnings_filters_mutated_impl() --- Python/_warnings.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index ac3d3cc2d1246b..4c520252aa12a8 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -398,9 +398,9 @@ already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, return -1; WarningsState *st = warnings_get_state(interp); - if (st == NULL) { - return -1; - } + assert(st != NULL); + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&st->mutex); + PyObject *version_obj; if (PyDict_GetItemRef(registry, &_Py_ID(version), &version_obj) < 0) { return -1; @@ -1177,11 +1177,14 @@ warnings_filters_mutated_impl(PyObject *module) if (interp == NULL) { return NULL; } + WarningsState *st = warnings_get_state(interp); - if (st == NULL) { - return NULL; - } + assert(st != NULL); + + Py_BEGIN_CRITICAL_SECTION_MUT(&st->mutex); st->filters_version++; + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; } From bfc57d43d8766120ba0c8f3f6d7b2ac681a81d8a Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 29 Mar 2024 18:58:08 -0400 Subject: [PATCH 024/143] gh-117303: Don't detach in `PyThreadState_DeleteCurrent()` (#117304) This fixes a crash in `test_threading.test_reinit_tls_after_fork()` when running with the GIL disabled. We already properly handle the case where the thread state is `_Py_THREAD_ATTACHED` in `tstate_delete_common()` -- we just need to remove an assertion. Keeping the thread attached means that a stop-the-world pause, such as for a `fork()`, won't commence until we remove our thread state from the interpreter's linked list. This prevents a crash when the child process tries to clean up the dead thread states. --- Python/pystate.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/pystate.c b/Python/pystate.c index 8bec72779b2c24..925d1cff866f18 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1660,7 +1660,6 @@ static void tstate_delete_common(PyThreadState *tstate) { assert(tstate->_status.cleared && !tstate->_status.finalized); - assert(tstate->state != _Py_THREAD_ATTACHED); tstate_verify_not_active(tstate); assert(!_PyThreadState_IsRunningMain(tstate)); @@ -1740,7 +1739,6 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) #ifdef Py_GIL_DISABLED _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); #endif - tstate_set_detached(tstate, _Py_THREAD_DETACHED); current_fast_clear(tstate->interp->runtime); tstate_delete_common(tstate); _PyEval_ReleaseLock(tstate->interp, NULL); From 752e18389ed03087b51b38eac9769ef8dfd167b7 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 31 Mar 2024 19:14:48 +0100 Subject: [PATCH 025/143] GH-114575: Rename `PurePath.pathmod` to `PurePath.parser` (#116513) And rename the private base class from `PathModuleBase` to `ParserBase`. --- Doc/library/pathlib.rst | 4 +- Doc/whatsnew/3.13.rst | 4 ++ Lib/pathlib/__init__.py | 54 ++++++++-------- Lib/pathlib/_abc.py | 50 +++++++-------- Lib/test/test_pathlib/test_pathlib.py | 54 ++++++++-------- Lib/test/test_pathlib/test_pathlib_abc.py | 76 +++++++++++------------ 6 files changed, 123 insertions(+), 119 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4fba3622b073a7..9122df7a476632 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -303,10 +303,10 @@ Methods and properties Pure paths provide the following methods and properties: -.. attribute:: PurePath.pathmod +.. attribute:: PurePath.parser The implementation of the :mod:`os.path` module used for low-level path - operations: either :mod:`posixpath` or :mod:`ntpath`. + parsing and joining: either :mod:`posixpath` or :mod:`ntpath`. .. versionadded:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index f50364a7ddcc2a..45f7f50bf9f46b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -524,6 +524,10 @@ pathlib shell-style wildcards, including the recursive wildcard "``**``". (Contributed by Barney Gale in :gh:`73435`.) +* Add :attr:`pathlib.PurePath.parser` class attribute that stores the + implementation of :mod:`os.path` used for low-level path parsing and + joining: either ``posixpath`` or ``ntpath``. + * Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.glob`, :meth:`~pathlib.Path.rglob`, :meth:`~pathlib.Path.is_file`, :meth:`~pathlib.Path.is_dir`, :meth:`~pathlib.Path.owner`, diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 46834b1a76a6eb..6cccfb864e8206 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -110,7 +110,7 @@ class PurePath(_abc.PurePathBase): # path. It's set when `__hash__()` is called for the first time. '_hash', ) - pathmod = os.path + parser = os.path def __new__(cls, *args, **kwargs): """Construct a PurePath from one or several strings and or existing @@ -126,7 +126,7 @@ def __init__(self, *args): paths = [] for arg in args: if isinstance(arg, PurePath): - if arg.pathmod is ntpath and self.pathmod is posixpath: + if arg.parser is ntpath and self.parser is posixpath: # GH-103631: Convert separators for backwards compatibility. paths.extend(path.replace('\\', '/') for path in arg._raw_paths) else: @@ -187,7 +187,7 @@ def _str_normcase(self): try: return self._str_normcase_cached except AttributeError: - if _abc._is_case_sensitive(self.pathmod): + if _abc._is_case_sensitive(self.parser): self._str_normcase_cached = str(self) else: self._str_normcase_cached = str(self).lower() @@ -203,7 +203,7 @@ def __hash__(self): def __eq__(self, other): if not isinstance(other, PurePath): return NotImplemented - return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod + return self._str_normcase == other._str_normcase and self.parser is other.parser @property def _parts_normcase(self): @@ -211,26 +211,26 @@ def _parts_normcase(self): try: return self._parts_normcase_cached except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) return self._parts_normcase_cached def __lt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + if not isinstance(other, PurePath) or self.parser is not other.parser: return NotImplemented return self._parts_normcase < other._parts_normcase def __le__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + if not isinstance(other, PurePath) or self.parser is not other.parser: return NotImplemented return self._parts_normcase <= other._parts_normcase def __gt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + if not isinstance(other, PurePath) or self.parser is not other.parser: return NotImplemented return self._parts_normcase > other._parts_normcase def __ge__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + if not isinstance(other, PurePath) or self.parser is not other.parser: return NotImplemented return self._parts_normcase >= other._parts_normcase @@ -247,10 +247,10 @@ def __str__(self): @classmethod def _format_parsed_parts(cls, drv, root, tail): if drv or root: - return drv + root + cls.pathmod.sep.join(tail) - elif tail and cls.pathmod.splitdrive(tail[0])[0]: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: tail = ['.'] + tail - return cls.pathmod.sep.join(tail) + return cls.parser.sep.join(tail) def _from_parsed_parts(self, drv, root, tail): path_str = self._format_parsed_parts(drv, root, tail) @@ -265,11 +265,11 @@ def _from_parsed_parts(self, drv, root, tail): def _parse_path(cls, path): if not path: return '', '', [] - sep = cls.pathmod.sep - altsep = cls.pathmod.altsep + sep = cls.parser.sep + altsep = cls.parser.altsep if altsep: path = path.replace(altsep, sep) - drv, root, rel = cls.pathmod.splitroot(path) + drv, root, rel = cls.parser.splitroot(path) if not root and drv.startswith(sep) and not drv.endswith(sep): drv_parts = drv.split(sep) if len(drv_parts) == 4 and drv_parts[2] not in '?.': @@ -290,7 +290,7 @@ def _raw_path(self): elif len(paths) == 1: path = paths[0] else: - path = self.pathmod.join(*paths) + path = self.parser.join(*paths) return path @property @@ -360,8 +360,8 @@ def name(self): def with_name(self, name): """Return a new path with the file name changed.""" - m = self.pathmod - if not name or m.sep in name or (m.altsep and m.altsep in name) or name == '.': + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': raise ValueError(f"Invalid name {name!r}") tail = self._tail.copy() if not tail: @@ -413,13 +413,13 @@ def is_relative_to(self, other, /, *_deprecated): def is_absolute(self): """True if the path is absolute (has both a root and, if applicable, a drive).""" - if self.pathmod is posixpath: + if self.parser is posixpath: # Optimization: work with raw paths on POSIX. for path in self._raw_paths: if path.startswith('/'): return True return False - return self.pathmod.isabs(self) + return self.parser.isabs(self) def is_reserved(self): """Return True if the path contains one of the special names reserved @@ -428,8 +428,8 @@ def is_reserved(self): "for removal in Python 3.15. Use os.path.isreserved() to " "detect reserved paths on Windows.") warnings.warn(msg, DeprecationWarning, stacklevel=2) - if self.pathmod is ntpath: - return self.pathmod.isreserved(self) + if self.parser is ntpath: + return self.parser.isreserved(self) return False def as_uri(self): @@ -462,7 +462,7 @@ def _pattern_stack(self): raise NotImplementedError("Non-relative patterns are unsupported") elif not parts: raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - elif pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): + elif pattern[-1] in (self.parser.sep, self.parser.altsep): # GH-65238: pathlib doesn't preserve trailing slash. Add it back. parts.append('') parts.reverse() @@ -487,7 +487,7 @@ class PurePosixPath(PurePath): On a POSIX system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ - pathmod = posixpath + parser = posixpath __slots__ = () @@ -497,7 +497,7 @@ class PureWindowsPath(PurePath): On a Windows system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ - pathmod = ntpath + parser = ntpath __slots__ = () @@ -607,7 +607,7 @@ def _make_child_relpath(self, name): path_str = str(self) tail = self._tail if tail: - path_str = f'{path_str}{self.pathmod.sep}{name}' + path_str = f'{path_str}{self.parser.sep}{name}' elif path_str != '.': path_str = f'{path_str}{name}' else: @@ -675,7 +675,7 @@ def absolute(self): drive, root, rel = os.path.splitroot(cwd) if not rel: return self._from_parsed_parts(drive, root, self._tail) - tail = rel.split(self.pathmod.sep) + tail = rel.split(self.parser.sep) tail.extend(self._tail) return self._from_parsed_parts(drive, root, tail) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 645d62a0f0699a..932020e6d0866c 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -37,8 +37,8 @@ def _ignore_error(exception): @functools.cache -def _is_case_sensitive(pathmod): - return pathmod.normcase('Aa') == 'Aa' +def _is_case_sensitive(parser): + return parser.normcase('Aa') == 'Aa' # # Globbing helpers @@ -156,12 +156,12 @@ class UnsupportedOperation(NotImplementedError): pass -class PathModuleBase: - """Base class for path modules, which do low-level path manipulation. +class ParserBase: + """Base class for path parsers, which do low-level path manipulation. - Path modules provide a subset of the os.path API, specifically those + Path parsers provide a subset of the os.path API, specifically those functions needed to provide PurePathBase functionality. Each PurePathBase - subclass references its path module via a 'pathmod' class attribute. + subclass references its path parser via a 'parser' class attribute. Every method in this base class raises an UnsupportedOperation exception. """ @@ -221,10 +221,10 @@ class PurePathBase: # work from occurring when `resolve()` calls `stat()` or `readlink()`. '_resolving', ) - pathmod = PathModuleBase() + parser = ParserBase() def __init__(self, path, *paths): - self._raw_path = self.pathmod.join(path, *paths) if paths else path + self._raw_path = self.parser.join(path, *paths) if paths else path if not isinstance(self._raw_path, str): raise TypeError( f"path should be a str, not {type(self._raw_path).__name__!r}") @@ -245,17 +245,17 @@ def __str__(self): def as_posix(self): """Return the string representation of the path with forward (/) slashes.""" - return str(self).replace(self.pathmod.sep, '/') + return str(self).replace(self.parser.sep, '/') @property def drive(self): """The drive prefix (letter or UNC path), if any.""" - return self.pathmod.splitdrive(self.anchor)[0] + return self.parser.splitdrive(self.anchor)[0] @property def root(self): """The root of the path, if any.""" - return self.pathmod.splitdrive(self.anchor)[1] + return self.parser.splitdrive(self.anchor)[1] @property def anchor(self): @@ -265,7 +265,7 @@ def anchor(self): @property def name(self): """The final path component, if any.""" - return self.pathmod.split(self._raw_path)[1] + return self.parser.split(self._raw_path)[1] @property def suffix(self): @@ -306,7 +306,7 @@ def stem(self): def with_name(self, name): """Return a new path with the file name changed.""" - split = self.pathmod.split + split = self.parser.split if split(name)[0]: raise ValueError(f"Invalid name {name!r}") return self.with_segments(split(self._raw_path)[0], name) @@ -419,7 +419,7 @@ def _stack(self): uppermost parent of the path (equivalent to path.parents[-1]), and *parts* is a reversed list of parts following the anchor. """ - split = self.pathmod.split + split = self.parser.split path = self._raw_path parent, name = split(path) names = [] @@ -433,7 +433,7 @@ def _stack(self): def parent(self): """The logical parent of the path.""" path = self._raw_path - parent = self.pathmod.split(path)[0] + parent = self.parser.split(path)[0] if path != parent: parent = self.with_segments(parent) parent._resolving = self._resolving @@ -443,7 +443,7 @@ def parent(self): @property def parents(self): """A sequence of this path's logical parents.""" - split = self.pathmod.split + split = self.parser.split path = self._raw_path parent = split(path)[0] parents = [] @@ -456,7 +456,7 @@ def parents(self): def is_absolute(self): """True if the path is absolute (has both a root and, if applicable, a drive).""" - return self.pathmod.isabs(self._raw_path) + return self.parser.isabs(self._raw_path) @property def _pattern_stack(self): @@ -481,8 +481,8 @@ def match(self, path_pattern, *, case_sensitive=None): if not isinstance(path_pattern, PurePathBase): path_pattern = self.with_segments(path_pattern) if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.pathmod) - sep = path_pattern.pathmod.sep + case_sensitive = _is_case_sensitive(self.parser) + sep = path_pattern.parser.sep path_parts = self.parts[::-1] pattern_parts = path_pattern.parts[::-1] if not pattern_parts: @@ -505,8 +505,8 @@ def full_match(self, pattern, *, case_sensitive=None): if not isinstance(pattern, PurePathBase): pattern = self.with_segments(pattern) if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.pathmod) - match = _compile_pattern(pattern._pattern_str, pattern.pathmod.sep, case_sensitive) + case_sensitive = _is_case_sensitive(self.parser) + match = _compile_pattern(pattern._pattern_str, pattern.parser.sep, case_sensitive) return match(self._pattern_str) is not None @@ -797,12 +797,12 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=True): pattern = self.with_segments(pattern) if case_sensitive is None: # TODO: evaluate case-sensitivity of each directory in _select_children(). - case_sensitive = _is_case_sensitive(self.pathmod) + case_sensitive = _is_case_sensitive(self.parser) stack = pattern._pattern_stack specials = ('', '.', '..') deduplicate_paths = False - sep = self.pathmod.sep + sep = self.parser.sep paths = iter([self] if self.is_dir() else []) while stack: part = stack.pop() @@ -973,7 +973,7 @@ def resolve(self, strict=False): continue path_tail.append(part) if querying and part != '..': - path = self.with_segments(path_root + self.pathmod.sep.join(path_tail)) + path = self.with_segments(path_root + self.parser.sep.join(path_tail)) path._resolving = True try: st = path.stat(follow_symlinks=False) @@ -1002,7 +1002,7 @@ def resolve(self, strict=False): raise else: querying = False - return self.with_segments(path_root + self.pathmod.sep.join(path_tail)) + return self.with_segments(path_root + self.parser.sep.join(path_tail)) def symlink_to(self, target, target_is_directory=False): """ diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 3a6f73c4fe82a4..651d66656cbd61 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -65,7 +65,7 @@ def test_concrete_class(self): p = self.cls('a') self.assertIs(type(p), expected) - def test_concrete_pathmod(self): + def test_concrete_parser(self): if self.cls is pathlib.PurePosixPath: expected = posixpath elif self.cls is pathlib.PureWindowsPath: @@ -73,19 +73,19 @@ def test_concrete_pathmod(self): else: expected = os.path p = self.cls('a') - self.assertIs(p.pathmod, expected) + self.assertIs(p.parser, expected) - def test_different_pathmods_unequal(self): + def test_different_parsers_unequal(self): p = self.cls('a') - if p.pathmod is posixpath: + if p.parser is posixpath: q = pathlib.PureWindowsPath('a') else: q = pathlib.PurePosixPath('a') self.assertNotEqual(p, q) - def test_different_pathmods_unordered(self): + def test_different_parsers_unordered(self): p = self.cls('a') - if p.pathmod is posixpath: + if p.parser is posixpath: q = pathlib.PureWindowsPath('a') else: q = pathlib.PurePosixPath('a') @@ -108,16 +108,16 @@ def test_constructor_nested(self): self.assertEqual(P(P('./a:b')), P('./a:b')) def _check_parse_path(self, raw_path, *expected): - sep = self.pathmod.sep + sep = self.parser.sep actual = self.cls._parse_path(raw_path.replace('/', sep)) self.assertEqual(actual, expected) - if altsep := self.pathmod.altsep: + if altsep := self.parser.altsep: actual = self.cls._parse_path(raw_path.replace('/', altsep)) self.assertEqual(actual, expected) def test_parse_path_common(self): check = self._check_parse_path - sep = self.pathmod.sep + sep = self.parser.sep check('', '', '', []) check('a', '', '', ['a']) check('a/', '', '', ['a']) @@ -523,10 +523,10 @@ class PathTest(test_pathlib_abc.DummyPathTest, PurePathTest): def setUp(self): super().setUp() - os.chmod(self.pathmod.join(self.base, 'dirE'), 0) + os.chmod(self.parser.join(self.base, 'dirE'), 0) def tearDown(self): - os.chmod(self.pathmod.join(self.base, 'dirE'), 0o777) + os.chmod(self.parser.join(self.base, 'dirE'), 0o777) os_helper.rmtree(self.base) def tempdir(self): @@ -541,8 +541,8 @@ def test_matches_pathbase_api(self): path_names = {name for name in dir(pathlib._abc.PathBase) if name[0] != '_'} self.assertEqual(our_names, path_names) for attr_name in our_names: - if attr_name == 'pathmod': - # On Windows, Path.pathmod is ntpath, but PathBase.pathmod is + if attr_name == 'parser': + # On Windows, Path.parser is ntpath, but PathBase.parser is # posixpath, and so their docstrings differ. continue our_attr = getattr(self.cls, attr_name) @@ -557,9 +557,9 @@ def test_concrete_class(self): p = self.cls('a') self.assertIs(type(p), expected) - def test_unsupported_pathmod(self): - if self.cls.pathmod is os.path: - self.skipTest("path flavour is supported") + def test_unsupported_parser(self): + if self.cls.parser is os.path: + self.skipTest("path parser is supported") else: self.assertRaises(pathlib.UnsupportedOperation, self.cls) @@ -809,7 +809,7 @@ def test_hardlink_to(self): self.assertTrue(target.exists()) # Linking to a str of a relative path. link2 = P / 'dirA' / 'fileAAA' - target2 = self.pathmod.join(TESTFN, 'fileA') + target2 = self.parser.join(TESTFN, 'fileA') link2.hardlink_to(target2) self.assertEqual(os.stat(target2).st_size, size) self.assertTrue(link2.exists()) @@ -834,7 +834,7 @@ def test_rename(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Renaming to a str of a relative path. - r = self.pathmod.join(TESTFN, 'fileAAA') + r = self.parser.join(TESTFN, 'fileAAA') renamed_q = q.rename(r) self.assertEqual(renamed_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) @@ -851,7 +851,7 @@ def test_replace(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Replacing another (existing) path. - r = self.pathmod.join(TESTFN, 'dirB', 'fileB') + r = self.parser.join(TESTFN, 'dirB', 'fileB') replaced_q = q.replace(r) self.assertEqual(replaced_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) @@ -1060,9 +1060,9 @@ def test_symlink_to_unsupported(self): def test_is_junction(self): P = self.cls(self.base) - with mock.patch.object(P.pathmod, 'isjunction'): - self.assertEqual(P.is_junction(), P.pathmod.isjunction.return_value) - P.pathmod.isjunction.assert_called_once_with(P) + with mock.patch.object(P.parser, 'isjunction'): + self.assertEqual(P.is_junction(), P.parser.isjunction.return_value) + P.parser.isjunction.assert_called_once_with(P) @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") @unittest.skipIf(sys.platform == "vxworks", @@ -1294,12 +1294,12 @@ def test_open_mode(self): p = self.cls(self.base) with (p / 'new_file').open('wb'): pass - st = os.stat(self.pathmod.join(self.base, 'new_file')) + st = os.stat(self.parser.join(self.base, 'new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) os.umask(0o022) with (p / 'other_new_file').open('wb'): pass - st = os.stat(self.pathmod.join(self.base, 'other_new_file')) + st = os.stat(self.parser.join(self.base, 'other_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) @needs_posix @@ -1322,14 +1322,14 @@ def test_touch_mode(self): self.addCleanup(os.umask, old_mask) p = self.cls(self.base) (p / 'new_file').touch() - st = os.stat(self.pathmod.join(self.base, 'new_file')) + st = os.stat(self.parser.join(self.base, 'new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) os.umask(0o022) (p / 'other_new_file').touch() - st = os.stat(self.pathmod.join(self.base, 'other_new_file')) + st = os.stat(self.parser.join(self.base, 'other_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) (p / 'masked_new_file').touch(mode=0o750) - st = os.stat(self.pathmod.join(self.base, 'masked_new_file')) + st = os.stat(self.parser.join(self.base, 'masked_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) @unittest.skipUnless(hasattr(pwd, 'getpwall'), diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 840fb903fd5338..a7e35a3e1fc7da 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -5,7 +5,7 @@ import stat import unittest -from pathlib._abc import UnsupportedOperation, PathModuleBase, PurePathBase, PathBase +from pathlib._abc import UnsupportedOperation, ParserBase, PurePathBase, PathBase import posixpath from test.support.os_helper import TESTFN @@ -38,8 +38,8 @@ def test_is_notimplemented(self): self.assertTrue(isinstance(UnsupportedOperation(), NotImplementedError)) -class PathModuleBaseTest(unittest.TestCase): - cls = PathModuleBase +class ParserBaseTest(unittest.TestCase): + cls = ParserBase def test_unsupported_operation(self): m = self.cls() @@ -109,13 +109,13 @@ def test_magic_methods(self): self.assertIs(P.__gt__, object.__gt__) self.assertIs(P.__ge__, object.__ge__) - def test_pathmod(self): - self.assertIsInstance(self.cls.pathmod, PathModuleBase) + def test_parser(self): + self.assertIsInstance(self.cls.parser, ParserBase) class DummyPurePath(PurePathBase): __slots__ = () - pathmod = posixpath + parser = posixpath def __eq__(self, other): if not isinstance(other, DummyPurePath): @@ -137,14 +137,14 @@ class DummyPurePathTest(unittest.TestCase): def setUp(self): name = self.id().split('.')[-1] - if name in _tests_needing_posix and self.cls.pathmod is not posixpath: + if name in _tests_needing_posix and self.cls.parser is not posixpath: self.skipTest('requires POSIX-flavoured path class') - if name in _tests_needing_windows and self.cls.pathmod is posixpath: + if name in _tests_needing_windows and self.cls.parser is posixpath: self.skipTest('requires Windows-flavoured path class') p = self.cls('a') - self.pathmod = p.pathmod - self.sep = self.pathmod.sep - self.altsep = self.pathmod.altsep + self.parser = p.parser + self.sep = self.parser.sep + self.altsep = self.parser.altsep def test_constructor_common(self): P = self.cls @@ -1411,7 +1411,7 @@ class DummyPath(PathBase): memory. """ __slots__ = () - pathmod = posixpath + parser = posixpath _files = {} _directories = {} @@ -1530,7 +1530,7 @@ def setUp(self): name = self.id().split('.')[-1] if name in _tests_needing_symlinks and not self.can_symlink: self.skipTest('requires symlinks') - pathmod = self.cls.pathmod + parser = self.cls.parser p = self.cls(self.base) p.mkdir(parents=True) p.joinpath('dirA').mkdir() @@ -1552,8 +1552,8 @@ def setUp(self): p.joinpath('linkA').symlink_to('fileA') p.joinpath('brokenLink').symlink_to('non-existing') p.joinpath('linkB').symlink_to('dirB') - p.joinpath('dirA', 'linkC').symlink_to(pathmod.join('..', 'dirB')) - p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB')) + p.joinpath('dirA', 'linkC').symlink_to(parser.join('..', 'dirB')) + p.joinpath('dirB', 'linkD').symlink_to(parser.join('..', 'dirB')) p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') def tearDown(self): @@ -1573,13 +1573,13 @@ def assertFileNotFound(self, func, *args, **kwargs): self.assertEqual(cm.exception.errno, errno.ENOENT) def assertEqualNormCase(self, path_a, path_b): - normcase = self.pathmod.normcase + normcase = self.parser.normcase self.assertEqual(normcase(path_a), normcase(path_b)) def test_samefile(self): - pathmod = self.pathmod - fileA_path = pathmod.join(self.base, 'fileA') - fileB_path = pathmod.join(self.base, 'dirB', 'fileB') + parser = self.parser + fileA_path = parser.join(self.base, 'fileA') + fileB_path = parser.join(self.base, 'dirB', 'fileB') p = self.cls(fileA_path) pp = self.cls(fileA_path) q = self.cls(fileB_path) @@ -1588,7 +1588,7 @@ def test_samefile(self): self.assertFalse(p.samefile(fileB_path)) self.assertFalse(p.samefile(q)) # Test the non-existent file case - non_existent = pathmod.join(self.base, 'foo') + non_existent = parser.join(self.base, 'foo') r = self.cls(non_existent) self.assertRaises(FileNotFoundError, p.samefile, r) self.assertRaises(FileNotFoundError, p.samefile, non_existent) @@ -2050,15 +2050,15 @@ def test_resolve_common(self): p.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ENOENT) # Non-strict - pathmod = self.pathmod + parser = self.parser self.assertEqualNormCase(str(p.resolve(strict=False)), - pathmod.join(self.base, 'foo')) + parser.join(self.base, 'foo')) p = P(self.base, 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - pathmod.join(self.base, 'foo', 'in', 'spam')) + parser.join(self.base, 'foo', 'in', 'spam')) p = P(self.base, '..', 'foo', 'in', 'spam') self.assertEqualNormCase(str(p.resolve(strict=False)), - pathmod.join(pathmod.dirname(self.base), 'foo', 'in', 'spam')) + parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) # These are all relative symlinks. p = P(self.base, 'dirB', 'fileB') self._check_resolve_relative(p, p) @@ -2073,7 +2073,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', 'spam'), False) p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if self.cls.pathmod is not posixpath: + if self.cls.parser is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', @@ -2085,7 +2085,7 @@ def test_resolve_common(self): # Now create absolute symlinks. d = self.tempdir() P(self.base, 'dirA', 'linkX').symlink_to(d) - P(self.base, str(d), 'linkY').symlink_to(self.pathmod.join(self.base, 'dirB')) + P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) # Non-strict @@ -2093,7 +2093,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), False) p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if self.cls.pathmod is not posixpath: + if self.cls.parser is not posixpath: # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) @@ -2105,11 +2105,11 @@ def test_resolve_common(self): @needs_symlinks def test_resolve_dot(self): # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - pathmod = self.pathmod + parser = self.parser p = self.cls(self.base) p.joinpath('0').symlink_to('.', target_is_directory=True) - p.joinpath('1').symlink_to(pathmod.join('0', '0'), target_is_directory=True) - p.joinpath('2').symlink_to(pathmod.join('1', '1'), target_is_directory=True) + p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) q = p / '2' self.assertEqual(q.resolve(strict=True), p) r = q / '3' / '4' @@ -2137,11 +2137,11 @@ def test_resolve_loop(self): p = self.cls(self.base, 'linkZ', 'foo') self.assertEqual(p.resolve(strict=False), p) # Loops with absolute symlinks. - self.cls(self.base, 'linkU').symlink_to(self.pathmod.join(self.base, 'linkU/inside')) + self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) self._check_symlink_loop(self.base, 'linkU') - self.cls(self.base, 'linkV').symlink_to(self.pathmod.join(self.base, 'linkV')) + self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) self._check_symlink_loop(self.base, 'linkV') - self.cls(self.base, 'linkW').symlink_to(self.pathmod.join(self.base, 'linkW/../linkW')) + self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) self._check_symlink_loop(self.base, 'linkW') # Non-strict q = self.cls(self.base, 'linkW', 'foo') @@ -2313,11 +2313,11 @@ def test_is_char_device_false(self): def _check_complex_symlinks(self, link0_target): # Test solving a non-looping chain of symlinks (issue #19887). - pathmod = self.pathmod + parser = self.parser P = self.cls(self.base) - P.joinpath('link1').symlink_to(pathmod.join('link0', 'link0'), target_is_directory=True) - P.joinpath('link2').symlink_to(pathmod.join('link1', 'link1'), target_is_directory=True) - P.joinpath('link3').symlink_to(pathmod.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) # Resolve absolute paths. @@ -2367,7 +2367,7 @@ def test_complex_symlinks_relative(self): @needs_symlinks def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(self.pathmod.join('dirA', '..')) + self._check_complex_symlinks(self.parser.join('dirA', '..')) def setUpWalk(self): # Build: From 262445358e21c56d7c68e3ee76c13e469d2ea348 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 31 Mar 2024 12:02:48 -0700 Subject: [PATCH 026/143] Link to the Python type system specification (#117400) --- Doc/library/typing.rst | 88 +++++++----------------------------------- 1 file changed, 15 insertions(+), 73 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 713ad1c83546d1..73214e18d556b2 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -23,27 +23,25 @@ -------------- -This module provides runtime support for type hints. For the original -specification of the typing system, see :pep:`484`. For a simplified -introduction to type hints, see :pep:`483`. +This module provides runtime support for type hints. +Consider the function below:: -The function below takes and returns a string and is annotated as follows:: + def moon_weight(earth_weight: float) -> str: + return f'On the moon, you would weigh {earth_weight * 0.166} kilograms.' - def greeting(name: str) -> str: - return 'Hello ' + name +The function ``moon_weight`` takes an argument expected to be an instance of :class:`float`, +as indicated by the *type hint* ``earth_weight: float``. The function is expected to +return an instance of :class:`str`, as indicated by the ``-> str`` hint. -In the function ``greeting``, the argument ``name`` is expected to be of type -:class:`str` and the return type :class:`str`. Subtypes are accepted as -arguments. +While type hints can be simple classes like :class:`float` or :class:`str`, +they can also be more complex. The :mod:`typing` module provides a vocabulary of +more advanced type hints. New features are frequently added to the ``typing`` module. The `typing_extensions `_ package provides backports of these new features to older versions of Python. -For a summary of deprecated features and a deprecation timeline, please see -`Deprecation Timeline of Major Features`_. - .. seealso:: `"Typing cheat sheet" `_ @@ -61,67 +59,11 @@ For a summary of deprecated features and a deprecation timeline, please see .. _relevant-peps: -Relevant PEPs -============= - -Since the initial introduction of type hints in :pep:`484` and :pep:`483`, a -number of PEPs have modified and enhanced Python's framework for type -annotations: - -.. raw:: html - -
- The full list of PEPs - -* :pep:`526`: Syntax for Variable Annotations - *Introducing* syntax for annotating variables outside of function - definitions, and :data:`ClassVar` -* :pep:`544`: Protocols: Structural subtyping (static duck typing) - *Introducing* :class:`Protocol` and the - :func:`@runtime_checkable` decorator -* :pep:`585`: Type Hinting Generics In Standard Collections - *Introducing* :class:`types.GenericAlias` and the ability to use standard - library classes as :ref:`generic types` -* :pep:`586`: Literal Types - *Introducing* :data:`Literal` -* :pep:`589`: TypedDict: Type Hints for Dictionaries with a Fixed Set of Keys - *Introducing* :class:`TypedDict` -* :pep:`591`: Adding a final qualifier to typing - *Introducing* :data:`Final` and the :func:`@final` decorator -* :pep:`593`: Flexible function and variable annotations - *Introducing* :data:`Annotated` -* :pep:`604`: Allow writing union types as ``X | Y`` - *Introducing* :data:`types.UnionType` and the ability to use - the binary-or operator ``|`` to signify a - :ref:`union of types` -* :pep:`612`: Parameter Specification Variables - *Introducing* :class:`ParamSpec` and :data:`Concatenate` -* :pep:`613`: Explicit Type Aliases - *Introducing* :data:`TypeAlias` -* :pep:`646`: Variadic Generics - *Introducing* :data:`TypeVarTuple` -* :pep:`647`: User-Defined Type Guards - *Introducing* :data:`TypeGuard` -* :pep:`655`: Marking individual TypedDict items as required or potentially missing - *Introducing* :data:`Required` and :data:`NotRequired` -* :pep:`673`: Self type - *Introducing* :data:`Self` -* :pep:`675`: Arbitrary Literal String Type - *Introducing* :data:`LiteralString` -* :pep:`681`: Data Class Transforms - *Introducing* the :func:`@dataclass_transform` decorator -* :pep:`692`: Using ``TypedDict`` for more precise ``**kwargs`` typing - *Introducing* a new way of typing ``**kwargs`` with :data:`Unpack` and - :data:`TypedDict` -* :pep:`695`: Type Parameter Syntax - *Introducing* builtin syntax for creating generic functions, classes, and type aliases. -* :pep:`698`: Adding an override decorator to typing - *Introducing* the :func:`@override` decorator - -.. raw:: html - -
-
+Specification for the Python Type System +======================================== + +The canonical, up-to-date specification of the Python type system can be +found at `"Specification for the Python type system" `_. .. _type-aliases: From a32d6939486d7f90ee57e215077f6116e19de24d Mon Sep 17 00:00:00 2001 From: Deborah <32307299+dlwrnc@users.noreply.github.com> Date: Sun, 31 Mar 2024 22:11:48 +0200 Subject: [PATCH 027/143] gh-102190: Add additional zipfile `pwd=` arg docstrings (gh-102195) This just documents the parameter that already exists. --------- Co-authored-by: Gregory P. Smith Co-authored-by: Erlend E. Aasland --- Lib/zipfile/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index b330ece4b276ad..e4603b559f5962 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -1585,7 +1585,8 @@ def comment(self, comment): self._didModify = True def read(self, name, pwd=None): - """Return file bytes for name.""" + """Return file bytes for name. 'pwd' is the password to decrypt + encrypted files.""" with self.open(name, "r", pwd) as fp: return fp.read() @@ -1737,7 +1738,8 @@ def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. + specify a different directory using `path'. You can specify the + password to decrypt the file using 'pwd'. """ if path is None: path = os.getcwd() @@ -1750,7 +1752,8 @@ def extractall(self, path=None, members=None, pwd=None): """Extract all members from the archive to the current working directory. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned - by namelist(). + by namelist(). You can specify the password to decrypt all files + using 'pwd'. """ if members is None: members = self.namelist() From 18e12641a61a88f7d08b2114ebe965892c6661c5 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 31 Mar 2024 16:09:22 -0500 Subject: [PATCH 028/143] gh-117387 Remove hash mark from introductory text (#117409) --- Lib/collections/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 0aa0c3e15e9519..2a35989ee25a5e 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -639,7 +639,8 @@ def elements(self): >>> sorted(c.elements()) ['A', 'A', 'B', 'B', 'C', 'C'] - # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + >>> import math >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) >>> math.prod(prime_factors.elements()) From 56e99307c49adcc6df355f8070229371c97d654f Mon Sep 17 00:00:00 2001 From: Adorilson Bezerra Date: Sun, 31 Mar 2024 23:34:54 +0100 Subject: [PATCH 029/143] Doc: printf-style library/stdtype improvements (#16741) --- Doc/library/stdtypes.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c963519f164dd2..62fc10997fc5b5 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2339,7 +2339,13 @@ String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. Given ``format % values`` (where *format* is a string), ``%`` conversion specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. +The effect is similar to using the :c:func:`sprintf` function in the C language. +For example: + +.. doctest:: + + >>> print('%s has %d quote types.' % ('Python', 2)) + Python has 2 quote types. If *format* requires a single argument, *values* may be a single non-tuple object. [5]_ Otherwise, *values* must be a tuple with exactly the number of From 3bb12e407c183946471272f8aee098e54e62a333 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:54:33 +0000 Subject: [PATCH 030/143] build(deps): bump actions/add-to-project from 0.6.0 to 1.0.0 (#117415) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/project-updater.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 56e508d453c346..066d8593a70cf6 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -23,7 +23,7 @@ jobs: - { project: 32, label: sprint } steps: - - uses: actions/add-to-project@v0.6.0 + - uses: actions/add-to-project@v1.0.0 with: project-url: https://github.com/orgs/python/projects/${{ matrix.project }} github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} From 93c7d9d17b571b6a181af7c02830d29819535c35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 11:38:38 +0100 Subject: [PATCH 031/143] build(deps-dev): bump types-setuptools from 69.1.0.20240301 to 69.2.0.20240317 in /Tools (#117419) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 2fb616e894a734..6944b15f7786fb 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -4,4 +4,4 @@ mypy==1.8.0 # needed for peg_generator: types-psutil==5.9.5.20240205 -types-setuptools==69.1.0.20240301 +types-setuptools==69.2.0.20240317 From 9b403fb559bfce93a478937e0ef7e539a9a95283 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 11:05:14 +0000 Subject: [PATCH 032/143] build(deps-dev): bump types-psutil from 5.9.5.20240205 to 5.9.5.20240316 in /Tools (#117417) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Waygood --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 6944b15f7786fb..fffd2246021547 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -3,5 +3,5 @@ mypy==1.8.0 # needed for peg_generator: -types-psutil==5.9.5.20240205 +types-psutil==5.9.5.20240316 types-setuptools==69.2.0.20240317 From 348cf6e0078eae156c503e8f61ef5e27ae28e57b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 11:14:37 +0000 Subject: [PATCH 033/143] Bump mypy from 1.8.0 to 1.9.0 in /Tools (#117418) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Waygood --- Lib/test/libregrtest/main.py | 4 +--- Tools/requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 70f723a92eb44a..3c9d9620053355 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -349,9 +349,7 @@ def run_test( namespace = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=namespace) result = namespace['result'] - # Mypy doesn't know about this attribute yet, - # but it will do soon: https://github.com/python/typeshed/pull/11091 - result.covered_lines = list(tracer.counts) # type: ignore[attr-defined] + result.covered_lines = list(tracer.counts) else: result = run_single_test(test_name, runtests) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index fffd2246021547..61e75cf396ccb4 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,6 +1,6 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.8.0 +mypy==1.9.0 # needed for peg_generator: types-psutil==5.9.5.20240316 From 90c3c68a658db6951b77a5be50088ec2f6adc8eb Mon Sep 17 00:00:00 2001 From: MonadChains Date: Mon, 1 Apr 2024 13:52:25 +0100 Subject: [PATCH 034/143] gh-94808:Improve coverage of PyObject_Print (GH-98749) --- Lib/test/test_capi/test_object.py | 55 +++++++++++- Lib/test/test_class.py | 1 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/object.c | 133 ++++++++++++++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 + PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + 8 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 Modules/_testcapi/object.c diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index c80e9b653789ad..fa23bff4e98918 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -1,8 +1,10 @@ import enum import unittest from test.support import import_helper +from test.support import os_helper _testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testcapi = import_helper.import_module('_testcapi') class Constant(enum.IntEnum): @@ -20,7 +22,7 @@ class Constant(enum.IntEnum): INVALID_CONSTANT = Py_CONSTANT_EMPTY_TUPLE + 1 -class CAPITest(unittest.TestCase): +class GetConstantTest(unittest.TestCase): def check_get_constant(self, get_constant): self.assertIs(get_constant(Constant.Py_CONSTANT_NONE), None) self.assertIs(get_constant(Constant.Py_CONSTANT_FALSE), False) @@ -50,5 +52,56 @@ def test_get_constant_borrowed(self): self.check_get_constant(_testlimitedcapi.get_constant_borrowed) +class PrintTest(unittest.TestCase): + def testPyObjectPrintObject(self): + + class PrintableObject: + + def __repr__(self): + return "spam spam spam" + + def __str__(self): + return "egg egg egg" + + obj = PrintableObject() + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.call_pyobject_print(obj, output_filename, False) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), repr(obj)) + + # Test str printing + _testcapi.call_pyobject_print(obj, output_filename, True) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), str(obj)) + + def testPyObjectPrintNULL(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.pyobject_print_null(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), '') + + def testPyObjectPrintNoRefObject(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + correct_output = _testcapi.pyobject_print_noref_object(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), correct_output) + + def testPyObjectPrintOSError(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + open(output_filename, "w+").close() + with self.assertRaises(OSError): + _testcapi.pyobject_print_os_error(output_filename) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index ad89a22c625dfd..0cf06243dc8da0 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -788,5 +788,6 @@ def __init__(self, obj): Type(i) self.assertEqual(calls, 100) + if __name__ == '__main__': unittest.main() diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index ff5c05f88d0d40..26720ef408fe3c 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -162,7 +162,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c new file mode 100644 index 00000000000000..ce5b574ec5ce8f --- /dev/null +++ b/Modules/_testcapi/object.c @@ -0,0 +1,133 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +call_pyobject_print(PyObject *self, PyObject * args) +{ + PyObject *object; + PyObject *filename; + PyObject *print_raw; + FILE *fp; + int flags = 0; + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 3, 3, + &object, &filename, &print_raw)) { + return NULL; + } + + fp = _Py_fopen_obj(filename, "w+"); + + if (Py_IsTrue(print_raw)) { + flags = Py_PRINT_RAW; + } + + if (PyObject_Print(object, fp, flags) < 0) { + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_print_null(PyObject *self, PyObject *args) +{ + PyObject *filename; + FILE *fp; + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + fp = _Py_fopen_obj(filename, "w+"); + + if (PyObject_Print(NULL, fp, 0) < 0) { + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_print_noref_object(PyObject *self, PyObject *args) +{ + PyObject *test_string; + PyObject *filename; + FILE *fp; + char correct_string[100]; + + test_string = PyUnicode_FromString("Spam spam spam"); + + Py_SET_REFCNT(test_string, 0); + + PyOS_snprintf(correct_string, 100, "", + Py_REFCNT(test_string), (void *)test_string); + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + fp = _Py_fopen_obj(filename, "w+"); + + if (PyObject_Print(test_string, fp, 0) < 0){ + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_SET_REFCNT(test_string, 1); + Py_DECREF(test_string); + + return PyUnicode_FromString(correct_string); +} + +static PyObject * +pyobject_print_os_error(PyObject *self, PyObject *args) +{ + PyObject *test_string; + PyObject *filename; + FILE *fp; + + test_string = PyUnicode_FromString("Spam spam spam"); + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + // open file in read mode to induce OSError + fp = _Py_fopen_obj(filename, "r"); + + if (PyObject_Print(test_string, fp, 0) < 0) { + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, + {"pyobject_print_null", pyobject_print_null, METH_VARARGS}, + {"pyobject_print_noref_object", pyobject_print_noref_object, METH_VARARGS}, + {"pyobject_print_os_error", pyobject_print_os_error, METH_VARARGS}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Object(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index e7c868f6bcff6e..2336cc0bc33a85 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -57,5 +57,6 @@ int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); int _PyTestCapi_Init_Hash(PyObject *module); int _PyTestCapi_Init_Time(PyObject *module); +int _PyTestCapi_Init_Object(PyObject *module); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3c30381be6d538..e9db6e5683e344 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4049,6 +4049,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Time(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Object(m) < 0) { + return NULL; + } PyState_AddModule(m, &_testcapimodule); return m; diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 615d73d5e003b4..afeb934b71b100 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -121,6 +121,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 0c11e918556ff5..b5bc4f36b2ff85 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -99,6 +99,9 @@ Source Files + + Source Files + Source Files From 3de09cadde788065a4f2d45117e789c9353bbd12 Mon Sep 17 00:00:00 2001 From: "Steve (Gadget) Barnes" Date: Mon, 1 Apr 2024 14:02:07 +0100 Subject: [PATCH 035/143] gh-91565: Replace bugs.python.org links with Devguide/GitHub ones (GH-91568) Co-authored-by: Hugo van Kemenade Co-authored-by: Oleg Iarygin Co-authored-by: Petr Viktorin Co-authored-by: Ezio Melotti --- Lib/locale.py | 3 ++- Lib/platform.py | 3 ++- Lib/test/crashers/README | 5 +++-- .../2022-04-15-13-15-23.gh-issue-91565.OznXwC.rst | 1 + Python/dtoa.c | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2022-04-15-13-15-23.gh-issue-91565.OznXwC.rst diff --git a/Lib/locale.py b/Lib/locale.py index e0cb4c5449d556..d8c09f1123d318 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -1460,7 +1460,8 @@ def getpreferredencoding(do_setlocale=True): # to include every locale up to Windows Vista. # # NOTE: this mapping is incomplete. If your language is missing, please -# submit a bug report to the Python bug tracker at http://bugs.python.org/ +# submit a bug report as detailed in the Python devguide at: +# https://devguide.python.org/triage/issue-tracker/ # Make sure you include the missing language identifier and the suggested # locale code. # diff --git a/Lib/platform.py b/Lib/platform.py index dbcb636df64981..ebaba37563120e 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -10,7 +10,8 @@ """ # This module is maintained by Marc-Andre Lemburg . # If you find problems, please submit bug reports/patches via the -# Python bug tracker (http://bugs.python.org) and assign them to "lemburg". +# Python issue tracker (https://github.com/python/cpython/issues) and +# mention "@malemburg". # # Still needed: # * support for MS-DOS (PythonDX ?) diff --git a/Lib/test/crashers/README b/Lib/test/crashers/README index 0259a0688cbc67..d844385113eb45 100644 --- a/Lib/test/crashers/README +++ b/Lib/test/crashers/README @@ -8,8 +8,9 @@ Each test should fail when run from the command line: ./python Lib/test/crashers/weakref_in_del.py Put as much info into a docstring or comments to help determine the cause of the -failure, as well as a bugs.python.org issue number if it exists. Particularly -note if the cause is system or environment dependent and what the variables are. +failure, as well as an issue number or link if it exists. +Particularly note if the cause is system or environment dependent and +what the variables are. Once the crash is fixed, the test case should be moved into an appropriate test (even if it was originally from the test suite). This ensures the regression diff --git a/Misc/NEWS.d/next/Documentation/2022-04-15-13-15-23.gh-issue-91565.OznXwC.rst b/Misc/NEWS.d/next/Documentation/2022-04-15-13-15-23.gh-issue-91565.OznXwC.rst new file mode 100644 index 00000000000000..df97e2c447ef58 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-04-15-13-15-23.gh-issue-91565.OznXwC.rst @@ -0,0 +1 @@ +Changes to documentation files and config outputs to reflect the new location for reporting bugs - i.e. GitHub rather than bugs.python.org. diff --git a/Python/dtoa.c b/Python/dtoa.c index 6e3162f80bdae1..8bba06d3b23289 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -72,7 +72,7 @@ /* Please send bug reports for the original dtoa.c code to David M. Gay (dmg * at acm dot org, with " at " changed at "@" and " dot " changed to "."). * Please report bugs for this modified version using the Python issue tracker - * (http://bugs.python.org). */ + * as detailed at (https://devguide.python.org/triage/issue-tracker/). */ /* On a machine with IEEE extended-precision registers, it is * necessary to specify double-precision (53-bit) rounding precision From dd44ab994b7262f0704d64996e0a1bc37b233407 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 1 Apr 2024 22:28:14 +0900 Subject: [PATCH 036/143] gh-117142: ctypes: Unify meta tp slot functions (GH-117143) Integrates the following ctypes meta tp slot functions: * `CDataType_traverse()` into `CType_Type_traverse()`. * `CDataType_clear()` into `CType_Type_clear()`. * `CDataType_dealloc()` into `CType_Type_dealloc()`. * `CDataType_repeat()` into `CType_Type_repeat()`. --- Modules/_ctypes/_ctypes.c | 109 +++++++++++++------------------------- Modules/_ctypes/ctypes.h | 8 +++ Modules/_ctypes/stgdict.c | 2 + 3 files changed, 46 insertions(+), 73 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 6bd1893480027c..631f82879311bf 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -441,12 +441,27 @@ static PyType_Spec structparam_spec = { static int CType_Type_traverse(PyObject *self, visitproc visit, void *arg) { + ctypes_state *st = GLOBAL_STATE(); + if (st && st->PyCType_Type) { + StgInfo *info; + if (PyStgInfo_FromType(st, self, &info) < 0) { + PyErr_WriteUnraisable(self); + } + if (info) { + Py_VISIT(info->proto); + Py_VISIT(info->argtypes); + Py_VISIT(info->converters); + Py_VISIT(info->restype); + Py_VISIT(info->checker); + Py_VISIT(info->module); + } + } Py_VISIT(Py_TYPE(self)); - return 0; + return PyType_Type.tp_traverse(self, visit, arg); } -static void -_ctype_clear_stginfo(StgInfo *info) +void +ctype_clear_stginfo(StgInfo *info) { assert(info); Py_CLEAR(info->proto); @@ -454,6 +469,7 @@ _ctype_clear_stginfo(StgInfo *info) Py_CLEAR(info->converters); Py_CLEAR(info->restype); Py_CLEAR(info->checker); + Py_CLEAR(info->module); } static int @@ -463,13 +479,13 @@ CType_Type_clear(PyObject *self) if (st && st->PyCType_Type) { StgInfo *info; if (PyStgInfo_FromType(st, self, &info) < 0) { - return -1; + PyErr_WriteUnraisable(self); } if (info) { - _ctype_clear_stginfo(info); + ctype_clear_stginfo(info); } } - return 0; + return PyType_Type.tp_clear(self); } static void @@ -489,7 +505,7 @@ CType_Type_dealloc(PyObject *self) info->format = NULL; PyMem_Free(info->shape); info->shape = NULL; - _ctype_clear_stginfo(info); + ctype_clear_stginfo(info); } } @@ -522,6 +538,10 @@ CType_Type_sizeof(PyObject *self) return PyLong_FromSsize_t(size); } +static PyObject * +CType_Type_repeat(PyObject *self, Py_ssize_t length); + + static PyMethodDef ctype_methods[] = { {"__sizeof__", _PyCFunction_CAST(CType_Type_sizeof), METH_NOARGS, PyDoc_STR("Return memory consumption of the type object.")}, @@ -533,6 +553,8 @@ static PyType_Slot ctype_type_slots[] = { {Py_tp_clear, CType_Type_clear}, {Py_tp_dealloc, CType_Type_dealloc}, {Py_tp_methods, ctype_methods}, + // Sequence protocol. + {Py_sq_repeat, CType_Type_repeat}, {0, NULL}, }; @@ -978,7 +1000,7 @@ static PyMethodDef CDataType_methods[] = { }; static PyObject * -CDataType_repeat(PyObject *self, Py_ssize_t length) +CType_Type_repeat(PyObject *self, Py_ssize_t length) { if (length < 0) return PyErr_Format(PyExc_ValueError, @@ -988,35 +1010,6 @@ CDataType_repeat(PyObject *self, Py_ssize_t length) return PyCArrayType_from_ctype(st, self, length); } -static int -CDataType_clear(PyTypeObject *self) -{ - ctypes_state *st = GLOBAL_STATE(); - StgInfo *info; - if (PyStgInfo_FromType(st, (PyObject *)self, &info) < 0) { - return -1; - } - if (info) { - Py_CLEAR(info->proto); - } - return PyType_Type.tp_clear((PyObject *)self); -} - -static int -CDataType_traverse(PyTypeObject *self, visitproc visit, void *arg) -{ - ctypes_state *st = GLOBAL_STATE(); - StgInfo *info; - if (PyStgInfo_FromType(st, (PyObject *)self, &info) < 0) { - return -1; - } - if (info) { - Py_VISIT(info->proto); - } - Py_VISIT(Py_TYPE(self)); - return PyType_Type.tp_traverse((PyObject *)self, visit, arg); -} - static int PyCStructType_setattro(PyObject *self, PyObject *key, PyObject *value) { @@ -1047,19 +1040,14 @@ UnionType_setattro(PyObject *self, PyObject *key, PyObject *value) static PyType_Slot pycstruct_type_slots[] = { {Py_tp_setattro, PyCStructType_setattro}, {Py_tp_doc, PyDoc_STR("metatype for the CData Objects")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, {Py_tp_init, PyCStructType_init}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, {0, NULL}, }; static PyType_Spec pycstruct_type_spec = { .name = "_ctypes.PyCStructType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycstruct_type_slots, }; @@ -1067,19 +1055,14 @@ static PyType_Spec pycstruct_type_spec = { static PyType_Slot union_type_slots[] = { {Py_tp_setattro, UnionType_setattro}, {Py_tp_doc, PyDoc_STR("metatype for the Union Objects")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, {Py_tp_init, UnionType_init}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, {0, NULL}, }; static PyType_Spec union_type_spec = { .name = "_ctypes.UnionType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = union_type_slots, }; @@ -1305,19 +1288,14 @@ static PyMethodDef PyCPointerType_methods[] = { static PyType_Slot pycpointer_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the Pointer Objects")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, PyCPointerType_methods}, {Py_tp_init, PyCPointerType_init}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, {0, NULL}, }; static PyType_Spec pycpointer_type_spec = { .name = "_ctypes.PyCPointerType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycpointer_type_slots, }; @@ -1640,19 +1618,14 @@ PyCArrayType_init(PyObject *self, PyObject *args, PyObject *kwds) static PyType_Slot pycarray_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the Array Objects")}, - {Py_tp_traverse, CDataType_traverse}, {Py_tp_methods, CDataType_methods}, {Py_tp_init, PyCArrayType_init}, - {Py_tp_clear, CDataType_clear}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, {0, NULL}, }; static PyType_Spec pycarray_type_spec = { .name = "_ctypes.PyCArrayType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycarray_type_slots, }; @@ -2315,17 +2288,12 @@ static PyType_Slot pycsimple_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the PyCSimpleType Objects")}, {Py_tp_methods, PyCSimpleType_methods}, {Py_tp_init, PyCSimpleType_init}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, {0, NULL}, }; static PyType_Spec pycsimple_type_spec = { .name = "_ctypes.PyCSimpleType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycsimple_type_slots, }; @@ -2570,19 +2538,14 @@ PyCFuncPtrType_init(PyObject *self, PyObject *args, PyObject *kwds) static PyType_Slot pycfuncptr_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for C function pointers")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, {Py_tp_init, PyCFuncPtrType_init}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, {0, NULL}, }; static PyType_Spec pycfuncptr_type_spec = { .name = "_ctypes.PyCFuncPtrType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycfuncptr_type_slots, }; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 3422310045bcc9..31b89dca244e8e 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -302,6 +302,7 @@ typedef struct { PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ PyObject *restype; /* CDataObject or NULL */ PyObject *checker; + PyObject *module; int flags; /* calling convention and such */ /* pep3118 fields, pointers need PyMem_Free */ @@ -313,6 +314,7 @@ typedef struct { } StgInfo; extern int PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info); +extern void ctype_clear_stginfo(StgInfo *info); typedef int(* PPROC)(void); @@ -481,6 +483,12 @@ PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) type->tp_name); return NULL; } + PyObject *module = PyType_GetModule(state->PyCType_Type); + if (!module) { + return NULL; + } + info->module = Py_NewRef(module); + info->initialized = 1; return info; } diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 53e7dc39614d21..7b09bae0dd2a57 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -25,6 +25,7 @@ PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) { Py_ssize_t size; + ctype_clear_stginfo(dst_info); PyMem_Free(dst_info->ffi_type_pointer.elements); PyMem_Free(dst_info->format); dst_info->format = NULL; @@ -39,6 +40,7 @@ PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) Py_XINCREF(dst_info->converters); Py_XINCREF(dst_info->restype); Py_XINCREF(dst_info->checker); + Py_XINCREF(dst_info->module); if (src_info->format) { dst_info->format = PyMem_Malloc(strlen(src_info->format) + 1); From 179869af922252a0c1cef65fd2923856895e7d1b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 1 Apr 2024 17:01:22 +0200 Subject: [PATCH 037/143] gh-94808: Fix refcounting in PyObject_Print tests (GH-117421) --- Modules/_testcapi/object.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index ce5b574ec5ce8f..8dd34cf4fc47d4 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -76,6 +76,8 @@ pyobject_print_noref_object(PyObject *self, PyObject *args) if (PyObject_Print(test_string, fp, 0) < 0){ fclose(fp); + Py_SET_REFCNT(test_string, 1); + Py_DECREF(test_string); return NULL; } @@ -105,10 +107,12 @@ pyobject_print_os_error(PyObject *self, PyObject *args) if (PyObject_Print(test_string, fp, 0) < 0) { fclose(fp); + Py_DECREF(test_string); return NULL; } fclose(fp); + Py_DECREF(test_string); Py_RETURN_NONE; } From ddf814db744006e0f42328aa15ace97c9d8ad681 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 1 Apr 2024 09:13:38 -0700 Subject: [PATCH 038/143] Silence compiler warnings in gc.c (#117422) --- Python/gc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/gc.c b/Python/gc.c index 36e20d05c205a5..a37c1b144e57e9 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1317,6 +1317,7 @@ gc_collect_young(PyThreadState *tstate, survivor_count++; } } + (void)survivor_count; // Silence compiler warning gc_list_merge(&survivors, visited); validate_old(gcstate); gcstate->young.count = 0; @@ -1329,12 +1330,14 @@ gc_collect_young(PyThreadState *tstate, add_stats(gcstate, 0, stats); } +#ifndef NDEBUG static inline int IS_IN_VISITED(PyGC_Head *gc, int visited_space) { assert(visited_space == 0 || flip_old_space(visited_space) == 0); return gc_old_space(gc) == visited_space; } +#endif struct container_and_flag { PyGC_Head *container; From c741ad3537193c63fe697a8f0316aecd45eeb9ba Mon Sep 17 00:00:00 2001 From: Justin Turner Arthur Date: Mon, 1 Apr 2024 12:07:29 -0500 Subject: [PATCH 039/143] gh-77714: Provide an async iterator version of as_completed (GH-22491) * as_completed returns object that is both iterator and async iterator * Existing tests adjusted to test both the old and new style * New test to ensure iterator can be resumed * New test to ensure async iterator yields any passed-in Futures as-is Co-authored-by: Serhiy Storchaka Co-authored-by: Guido van Rossum --- Doc/library/asyncio-task.rst | 61 +++- Doc/whatsnew/3.13.rst | 7 + Lib/asyncio/tasks.py | 152 +++++++--- Lib/test/test_asyncio/test_tasks.py | 282 ++++++++++++++---- .../2020-10-02-17-35-19.bpo-33533.GLIhM5.rst | 5 + 5 files changed, 387 insertions(+), 120 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-10-02-17-35-19.bpo-33533.GLIhM5.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 2aab62c64d2920..3b10a0d628a86e 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -867,19 +867,50 @@ Waiting Primitives .. function:: as_completed(aws, *, timeout=None) - Run :ref:`awaitable objects ` in the *aws* - iterable concurrently. Return an iterator of coroutines. - Each coroutine returned can be awaited to get the earliest next - result from the iterable of the remaining awaitables. - - Raises :exc:`TimeoutError` if the timeout occurs before - all Futures are done. - - Example:: - - for coro in as_completed(aws): - earliest_result = await coro - # ... + Run :ref:`awaitable objects ` in the *aws* iterable + concurrently. The returned object can be iterated to obtain the results + of the awaitables as they finish. + + The object returned by ``as_completed()`` can be iterated as an + :term:`asynchronous iterator` or a plain :term:`iterator`. When asynchronous + iteration is used, the originally-supplied awaitables are yielded if they + are tasks or futures. This makes it easy to correlate previously-scheduled + tasks with their results. Example:: + + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] + + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect + + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") + + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. + + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13:: + + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] + + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect + + A :exc:`TimeoutError` is raised if the timeout occurs before all awaitables + are done. This is raised by the ``async for`` loop during asynchronous + iteration or by the coroutines yielded during plain iteration. .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -891,6 +922,10 @@ Waiting Primitives .. versionchanged:: 3.12 Added support for generators yielding tasks. + .. versionchanged:: 3.13 + The result can now be used as either an :term:`asynchronous iterator` + or as a plain :term:`iterator` (previously it was only a plain iterator). + Running in Threads ================== diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 45f7f50bf9f46b..97bee4d38e300a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -289,6 +289,13 @@ asyncio forcefully close an asyncio server. (Contributed by Pierre Ossman in :gh:`113538`.) +* :func:`asyncio.as_completed` now returns an object that is both an + :term:`asynchronous iterator` and a plain :term:`iterator` of awaitables. + The awaitables yielded by asynchronous iteration include original task or + future objects that were passed in, making it easier to associate results + with the tasks being completed. + (Contributed by Justin Arthur in :gh:`77714`.) + base64 ------ diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 48e31af9a43167..7fb697b9441c33 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -25,6 +25,7 @@ from . import events from . import exceptions from . import futures +from . import queues from . import timeouts # Helper to generate new task names @@ -564,62 +565,125 @@ async def _cancel_and_wait(fut): fut.remove_done_callback(cb) -# This is *not* a @coroutine! It is just an iterator (yielding Futures). +class _AsCompletedIterator: + """Iterator of awaitables representing tasks of asyncio.as_completed. + + As an asynchronous iterator, iteration yields futures as they finish. As a + plain iterator, new coroutines are yielded that will return or raise the + result of the next underlying future to complete. + """ + def __init__(self, aws, timeout): + self._done = queues.Queue() + self._timeout_handle = None + + loop = events.get_event_loop() + todo = {ensure_future(aw, loop=loop) for aw in set(aws)} + for f in todo: + f.add_done_callback(self._handle_completion) + if todo and timeout is not None: + self._timeout_handle = ( + loop.call_later(timeout, self._handle_timeout) + ) + self._todo = todo + self._todo_left = len(todo) + + def __aiter__(self): + return self + + def __iter__(self): + return self + + async def __anext__(self): + if not self._todo_left: + raise StopAsyncIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return await self._wait_for_one() + + def __next__(self): + if not self._todo_left: + raise StopIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return self._wait_for_one(resolve=True) + + def _handle_timeout(self): + for f in self._todo: + f.remove_done_callback(self._handle_completion) + self._done.put_nowait(None) # Sentinel for _wait_for_one(). + self._todo.clear() # Can't do todo.remove(f) in the loop. + + def _handle_completion(self, f): + if not self._todo: + return # _handle_timeout() was here first. + self._todo.remove(f) + self._done.put_nowait(f) + if not self._todo and self._timeout_handle is not None: + self._timeout_handle.cancel() + + async def _wait_for_one(self, resolve=False): + # Wait for the next future to be done and return it unless resolve is + # set, in which case return either the result of the future or raise + # an exception. + f = await self._done.get() + if f is None: + # Dummy value from _handle_timeout(). + raise exceptions.TimeoutError + return f.result() if resolve else f + + def as_completed(fs, *, timeout=None): - """Return an iterator whose values are coroutines. + """Create an iterator of awaitables or their results in completion order. - When waiting for the yielded coroutines you'll get the results (or - exceptions!) of the original Futures (or coroutines), in the order - in which and as soon as they complete. + Run the supplied awaitables concurrently. The returned object can be + iterated to obtain the results of the awaitables as they finish. - This differs from PEP 3148; the proper way to use this is: + The object returned can be iterated as an asynchronous iterator or a plain + iterator. When asynchronous iteration is used, the originally-supplied + awaitables are yielded if they are tasks or futures. This makes it easy to + correlate previously-scheduled tasks with their results: - for f in as_completed(fs): - result = await f # The 'await' may raise. - # Use result. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - If a timeout is specified, the 'await' will raise - TimeoutError when the timeout occurs before all Futures are done. + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect - Note: The futures 'f' are not necessarily members of fs. - """ - if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError(f"expect an iterable of futures, not {type(fs).__name__}") + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") - from .queues import Queue # Import here to avoid circular import problem. - done = Queue() + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. - loop = events.get_event_loop() - todo = {ensure_future(f, loop=loop) for f in set(fs)} - timeout_handle = None + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13: - def _on_timeout(): - for f in todo: - f.remove_done_callback(_on_completion) - done.put_nowait(None) # Queue a dummy value for _wait_for_one(). - todo.clear() # Can't do todo.remove(f) in the loop. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - def _on_completion(f): - if not todo: - return # _on_timeout() was here first. - todo.remove(f) - done.put_nowait(f) - if not todo and timeout_handle is not None: - timeout_handle.cancel() + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect - async def _wait_for_one(): - f = await done.get() - if f is None: - # Dummy value from _on_timeout(). - raise exceptions.TimeoutError - return f.result() # May raise f.exception(). + A TimeoutError is raised if the timeout occurs before all awaitables are + done. This is raised by the async for loop during asynchronous iteration or + by the coroutines yielded during plain iteration. + """ + if inspect.isawaitable(fs): + raise TypeError( + f"expects an iterable of awaitables, not {type(fs).__name__}" + ) - for f in todo: - f.add_done_callback(_on_completion) - if todo and timeout is not None: - timeout_handle = loop.call_later(timeout, _on_timeout) - for _ in range(len(todo)): - yield _wait_for_one() + return _AsCompletedIterator(fs, timeout) @types.coroutine diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 4dfaff847edb90..bc6d88e65a4966 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1,6 +1,7 @@ """Tests for tasks.py.""" import collections +import contextlib import contextvars import gc import io @@ -1409,12 +1410,6 @@ def gen(): yield 0.01 yield 0 - loop = self.new_test_loop(gen) - # disable "slow callback" warning - loop.slow_callback_duration = 1.0 - completed = set() - time_shifted = False - async def sleeper(dt, x): nonlocal time_shifted await asyncio.sleep(dt) @@ -1424,21 +1419,78 @@ async def sleeper(dt, x): loop.advance_time(0.14) return x - a = sleeper(0.01, 'a') - b = sleeper(0.01, 'b') - c = sleeper(0.15, 'c') + async def try_iterator(awaitables): + values = [] + for f in asyncio.as_completed(awaitables): + values.append(await f) + return values - async def foo(): + async def try_async_iterator(awaitables): values = [] - for f in asyncio.as_completed([b, c, a]): + async for f in asyncio.as_completed(awaitables): values.append(await f) return values - res = loop.run_until_complete(self.new_task(loop, foo())) - self.assertAlmostEqual(0.15, loop.time()) - self.assertTrue('a' in res[:2]) - self.assertTrue('b' in res[:2]) - self.assertEqual(res[2], 'c') + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + # disable "slow callback" warning + loop.slow_callback_duration = 1.0 + + completed = set() + time_shifted = False + + a = sleeper(0.01, 'a') + b = sleeper(0.01, 'b') + c = sleeper(0.15, 'c') + + res = loop.run_until_complete(self.new_task(loop, foo([b, c, a]))) + self.assertAlmostEqual(0.15, loop.time()) + self.assertTrue('a' in res[:2]) + self.assertTrue('b' in res[:2]) + self.assertEqual(res[2], 'c') + + def test_as_completed_same_tasks_in_as_out(self): + # Ensures that asynchronously iterating as_completed's iterator + # yields awaitables are the same awaitables that were passed in when + # those awaitables are futures. + async def try_async_iterator(awaitables): + awaitables_out = set() + async for out_aw in asyncio.as_completed(awaitables): + awaitables_out.add(out_aw) + return awaitables_out + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + # Coroutines shouldn't be yielded back as finished coroutines + # can't be re-used. + awaitables_in = frozenset( + (coro(0), coro(1), coro(2), coro(3)) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in - awaitables_out != awaitables_in: + raise self.failureException('Got original coroutines ' + 'out of as_completed iterator.') + + # Tasks should be yielded back. + coro_obj_a = coro('a') + task_b = loop.create_task(coro('b')) + coro_obj_c = coro('c') + task_d = loop.create_task(coro('d')) + awaitables_in = frozenset( + (coro_obj_a, task_b, coro_obj_c, task_d) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in & awaitables_out != {task_b, task_d}: + raise self.failureException('Only tasks should be yielded ' + 'from as_completed iterator ' + 'as-is.') def test_as_completed_with_timeout(self): @@ -1448,12 +1500,7 @@ def gen(): yield 0 yield 0.1 - loop = self.new_test_loop(gen) - - a = loop.create_task(asyncio.sleep(0.1, 'a')) - b = loop.create_task(asyncio.sleep(0.15, 'b')) - - async def foo(): + async def try_iterator(): values = [] for f in asyncio.as_completed([a, b], timeout=0.12): if values: @@ -1465,16 +1512,33 @@ async def foo(): values.append((2, exc)) return values - res = loop.run_until_complete(self.new_task(loop, foo())) - self.assertEqual(len(res), 2, res) - self.assertEqual(res[0], (1, 'a')) - self.assertEqual(res[1][0], 2) - self.assertIsInstance(res[1][1], asyncio.TimeoutError) - self.assertAlmostEqual(0.12, loop.time()) + async def try_async_iterator(): + values = [] + try: + async for f in asyncio.as_completed([a, b], timeout=0.12): + v = await f + values.append((1, v)) + loop.advance_time(0.02) + except asyncio.TimeoutError as exc: + values.append((2, exc)) + return values - # move forward to close generator - loop.advance_time(10) - loop.run_until_complete(asyncio.wait([a, b])) + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + a = loop.create_task(asyncio.sleep(0.1, 'a')) + b = loop.create_task(asyncio.sleep(0.15, 'b')) + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(len(res), 2, res) + self.assertEqual(res[0], (1, 'a')) + self.assertEqual(res[1][0], 2) + self.assertIsInstance(res[1][1], asyncio.TimeoutError) + self.assertAlmostEqual(0.12, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) def test_as_completed_with_unused_timeout(self): @@ -1483,19 +1547,75 @@ def gen(): yield 0 yield 0.01 - loop = self.new_test_loop(gen) - - a = asyncio.sleep(0.01, 'a') - - async def foo(): + async def try_iterator(): for f in asyncio.as_completed([a], timeout=1): v = await f self.assertEqual(v, 'a') - loop.run_until_complete(self.new_task(loop, foo())) + async def try_async_iterator(): + async for f in asyncio.as_completed([a], timeout=1): + v = await f + self.assertEqual(v, 'a') - def test_as_completed_reverse_wait(self): + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + a = asyncio.sleep(0.01, 'a') + loop = self.new_test_loop(gen) + loop.run_until_complete(self.new_task(loop, foo())) + loop.close() + + def test_as_completed_resume_iterator(self): + # Test that as_completed returns an iterator that can be resumed + # the next time iteration is performed (i.e. if __iter__ is called + # again) + async def try_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + for f in iterator: + collected.append(await f) + return collected + + async def try_async_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + async for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + async for f in iterator: + collected.append(await f) + return collected + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + results = loop.run_until_complete( + foo((coro(0), coro(1), coro(2), coro(3))) + ) + self.assertCountEqual(results, (0, 1, 2, 3)) + def test_as_completed_reverse_wait(self): + # Tests the plain iterator style of as_completed iteration to + # ensure that the first future awaited resolves to the first + # completed awaitable from the set we passed in, even if it wasn't + # the first future generated by as_completed. def gen(): yield 0 yield 0.05 @@ -1522,7 +1642,8 @@ async def test(): loop.run_until_complete(test()) def test_as_completed_concurrent(self): - + # Ensure that more than one future or coroutine yielded from + # as_completed can be awaited concurrently. def gen(): when = yield self.assertAlmostEqual(0.05, when) @@ -1530,38 +1651,55 @@ def gen(): self.assertAlmostEqual(0.05, when) yield 0.05 - a = asyncio.sleep(0.05, 'a') - b = asyncio.sleep(0.05, 'b') - fs = {a, b} + async def try_iterator(fs): + return list(asyncio.as_completed(fs)) - async def test(): - futs = list(asyncio.as_completed(fs)) - self.assertEqual(len(futs), 2) - done, pending = await asyncio.wait( - [asyncio.ensure_future(fut) for fut in futs] - ) - self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + async def try_async_iterator(fs): + return [f async for f in asyncio.as_completed(fs)] - loop = self.new_test_loop(gen) - loop.run_until_complete(test()) + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + a = asyncio.sleep(0.05, 'a') + b = asyncio.sleep(0.05, 'b') + fs = {a, b} + + async def test(): + futs = await runner(fs) + self.assertEqual(len(futs), 2) + done, pending = await asyncio.wait( + [asyncio.ensure_future(fut) for fut in futs] + ) + self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + + loop = self.new_test_loop(gen) + loop.run_until_complete(test()) def test_as_completed_duplicate_coroutines(self): async def coro(s): return s - async def runner(): + async def try_iterator(): result = [] c = coro('ham') for f in asyncio.as_completed([c, c, coro('spam')]): result.append(await f) return result - fut = self.new_task(self.loop, runner()) - self.loop.run_until_complete(fut) - result = fut.result() - self.assertEqual(set(result), {'ham', 'spam'}) - self.assertEqual(len(result), 2) + async def try_async_iterator(): + result = [] + c = coro('ham') + async for f in asyncio.as_completed([c, c, coro('spam')]): + result.append(await f) + return result + + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + fut = self.new_task(self.loop, runner()) + self.loop.run_until_complete(fut) + result = fut.result() + self.assertEqual(set(result), {'ham', 'spam'}) + self.assertEqual(len(result), 2) def test_as_completed_coroutine_without_loop(self): async def coro(): @@ -1570,8 +1708,8 @@ async def coro(): a = coro() self.addCleanup(a.close) - futs = asyncio.as_completed([a]) with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + futs = asyncio.as_completed([a]) list(futs) def test_as_completed_coroutine_use_running_loop(self): @@ -2044,14 +2182,32 @@ async def coro(): self.assertEqual(res, 42) def test_as_completed_invalid_args(self): + # as_completed() expects a list of futures, not a future instance + # TypeError should be raised either on iterator construction or first + # iteration + + # Plain iterator fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(fut) + next(iterator) + coro = coroutine_function() + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(coro) + next(iterator) + coro.close() - # as_completed() expects a list of futures, not a future instance - self.assertRaises(TypeError, self.loop.run_until_complete, - asyncio.as_completed(fut)) + # Async iterator + async def try_async_iterator(aw): + async for f in asyncio.as_completed(aw): + break + + fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(fut)) coro = coroutine_function() - self.assertRaises(TypeError, self.loop.run_until_complete, - asyncio.as_completed(coro)) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(coro)) coro.close() def test_wait_invalid_args(self): diff --git a/Misc/NEWS.d/next/Library/2020-10-02-17-35-19.bpo-33533.GLIhM5.rst b/Misc/NEWS.d/next/Library/2020-10-02-17-35-19.bpo-33533.GLIhM5.rst new file mode 100644 index 00000000000000..3ffd723cf1082a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-02-17-35-19.bpo-33533.GLIhM5.rst @@ -0,0 +1,5 @@ +:func:`asyncio.as_completed` now returns an object that is both an asynchronous +iterator and plain iterator. The new asynchronous iteration pattern allows for +easier correlation between prior tasks and their completed results. This is +a closer match to :func:`concurrent.futures.as_completed`'s iteration pattern. +Patch by Justin Arthur. From fc8007ee3635db6ab73e132ebff987c910b6d538 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Mon, 1 Apr 2024 20:37:41 +0100 Subject: [PATCH 040/143] GH-117337: Deprecate `glob.glob0()` and `glob.glob1()`. (#117371) These undocumented functions are no longer used by `msilib`, so there's no reason to keep them around. --- Doc/whatsnew/3.13.rst | 5 +++ Lib/glob.py | 9 +++++- Lib/test/test_glob.py | 31 +++++++++++++++++++ ...-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst | 3 ++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 97bee4d38e300a..7f6a86efc61bf7 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -813,6 +813,11 @@ Deprecated translation was not found. (Contributed by Serhiy Storchaka in :gh:`88434`.) +* :mod:`glob`: The undocumented :func:`!glob.glob0` and :func:`!glob.glob1` + functions are deprecated. Use :func:`glob.glob` and pass a directory to its + *root_dir* argument instead. + (Contributed by Barney Gale in :gh:`117337`.) + * :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI HTTP servers have been out of favor for a very long time. This code was diff --git a/Lib/glob.py b/Lib/glob.py index d59641195a1c41..a915cf0bdf4502 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -119,12 +119,19 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -# Following functions are not public but can be used by third-party code. +_deprecated_function_message = ( + "{name} is deprecated and will be removed in Python {remove}. Use " + "glob.glob and pass a directory to its root_dir argument instead." +) def glob0(dirname, pattern): + import warnings + warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) return _glob0(dirname, pattern, None, False) def glob1(dirname, pattern): + import warnings + warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) return _glob1(dirname, pattern, None, False) # This helper function recursively yields relative pathnames inside a literal diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 6719bdbb0cc9b1..70ee35ed2850bc 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -4,6 +4,7 @@ import shutil import sys import unittest +import warnings from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -382,6 +383,36 @@ def test_glob_many_open_files(self): for it in iters: self.assertEqual(next(it), p) + def test_glob0(self): + with self.assertWarns(DeprecationWarning): + glob.glob0(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob0(self.tempdir, 'a'), ['a']) + eq(glob.glob0(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob0(self.tempdir, '.b*'), []) + eq(glob.glob0(self.tempdir, 'b'), []) + eq(glob.glob0(self.tempdir, '?'), []) + eq(glob.glob0(self.tempdir, '*a'), []) + eq(glob.glob0(self.tempdir, 'a*'), []) + + def test_glob1(self): + with self.assertWarns(DeprecationWarning): + glob.glob1(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob1(self.tempdir, 'a'), ['a']) + eq(glob.glob1(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob1(self.tempdir, '.b*'), ['.bb']) + eq(glob.glob1(self.tempdir, 'b'), []) + eq(glob.glob1(self.tempdir, '?'), ['a']) + eq(glob.glob1(self.tempdir, '*a'), ['a', 'aaa']) + eq(glob.glob1(self.tempdir, 'a*'), ['a', 'aaa', 'aab']) + def test_translate_matching(self): match = re.compile(glob.translate('*')).match self.assertIsNotNone(match('foo')) diff --git a/Misc/NEWS.d/next/Library/2024-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst b/Misc/NEWS.d/next/Library/2024-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst new file mode 100644 index 00000000000000..73bd2569c7c9cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst @@ -0,0 +1,3 @@ +Deprecate undocumented :func:`!glob.glob0` and :func:`!glob.glob1` +functions. Use :func:`glob.glob` and pass a directory to its +*root_dir* argument instead. From fc2071687b708598264a3403b7f9104667c1092f Mon Sep 17 00:00:00 2001 From: Matthew Davis <7035647+mdavis-xyz@users.noreply.github.com> Date: Mon, 1 Apr 2024 21:49:14 +0200 Subject: [PATCH 041/143] Docs: add more links to PIPE in subprocess docs (#25416) --- Doc/library/subprocess.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 8f6751cb11af2a..49194b82b4cea2 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -52,10 +52,12 @@ underlying :class:`Popen` interface can be used directly. If *capture_output* is true, stdout and stderr will be captured. When used, the internal :class:`Popen` object is automatically created with - ``stdout=PIPE`` and ``stderr=PIPE``. The *stdout* and *stderr* arguments may - not be supplied at the same time as *capture_output*. If you wish to capture - and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT`` - instead of *capture_output*. + *stdout* and *stdin* both set to :data:`~subprocess.PIPE`. + The *stdout* and *stderr* arguments may not be supplied at the same time as *capture_output*. + If you wish to capture and combine both streams into one, + set *stdout* to :data:`~subprocess.PIPE` + and *stderr* to :data:`~subprocess.STDOUT`, + instead of using *capture_output*. A *timeout* may be specified in seconds, it is internally passed on to :meth:`Popen.communicate`. If the timeout expires, the child process will be @@ -69,7 +71,8 @@ underlying :class:`Popen` interface can be used directly. subprocess's stdin. If used it must be a byte sequence, or a string if *encoding* or *errors* is specified or *text* is true. When used, the internal :class:`Popen` object is automatically created with - ``stdin=PIPE``, and the *stdin* argument may not be used as well. + *stdin* set to :data:`~subprocess.PIPE`, + and the *stdin* argument may not be used as well. If *check* is true, and the process exits with a non-zero exit code, a :exc:`CalledProcessError` exception will be raised. Attributes of that From 9dae05ee59eeba0e67af2a46f2a2907c9f8d7e4a Mon Sep 17 00:00:00 2001 From: Moshe Kaplan Date: Mon, 1 Apr 2024 15:53:00 -0400 Subject: [PATCH 042/143] Docs: specify XML document name in xml.etree.elementtree example (#24223) --- Doc/library/xml.etree.elementtree.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 5955647588fa3e..7d721f7633899e 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -49,7 +49,7 @@ and its sub-elements are done on the :class:`Element` level. Parsing XML ^^^^^^^^^^^ -We'll be using the following XML document as the sample data for this section: +We'll be using the fictive :file:`country_data.xml` XML document as the sample data for this section: .. code-block:: xml From 5fd1897ec51cb64ef7990ada538fcd8d9ca1f74b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 2 Apr 2024 12:09:53 +0200 Subject: [PATCH 043/143] gh-113317: Argument Clinic: Add libclinic.converters module (#117315) Move the following converter classes to libclinic.converters: * PyByteArrayObject_converter * PyBytesObject_converter * Py_UNICODE_converter * Py_buffer_converter * Py_complex_converter * Py_ssize_t_converter * bool_converter * byte_converter * char_converter * defining_class_converter * double_converter * fildes_converter * float_converter * int_converter * long_converter * long_long_converter * object_converter * self_converter * short_converter * size_t_converter * slice_index_converter * str_converter * unicode_converter * unsigned_char_converter * unsigned_int_converter * unsigned_long_converter * unsigned_long_long_converter * unsigned_short_converter Move also the following classes to libclinic.converters: * buffer * robuffer * rwbuffer Move the following functions to libclinic.converters: * correct_name_for_self() * r() * str_converter_key() Move Null and NULL to libclinic.utils. --- Lib/test/test_clinic.py | 7 +- Tools/clinic/clinic.py | 1208 +------------------------ Tools/clinic/libclinic/__init__.py | 20 +- Tools/clinic/libclinic/converter.py | 16 + Tools/clinic/libclinic/converters.py | 1211 ++++++++++++++++++++++++++ Tools/clinic/libclinic/function.py | 3 +- Tools/clinic/libclinic/utils.py | 9 + 7 files changed, 1259 insertions(+), 1215 deletions(-) create mode 100644 Tools/clinic/libclinic/converters.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f95bf858100be6..a07641d1dfda54 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -17,6 +17,7 @@ test_tools.skip_if_missing('clinic') with test_tools.imports_under_tool('clinic'): import libclinic + from libclinic.converters import int_converter, str_converter import clinic from clinic import DSLParser @@ -924,7 +925,7 @@ def test_param(self): self.assertEqual(2, len(function.parameters)) p = function.parameters['path'] self.assertEqual('path', p.name) - self.assertIsInstance(p.converter, clinic.int_converter) + self.assertIsInstance(p.converter, int_converter) def test_param_default(self): function = self.parse_function(""" @@ -1023,7 +1024,7 @@ def test_param_no_docstring(self): """) self.assertEqual(3, len(function.parameters)) conv = function.parameters['something_else'].converter - self.assertIsInstance(conv, clinic.str_converter) + self.assertIsInstance(conv, str_converter) def test_param_default_parameters_out_of_order(self): err = ( @@ -2040,7 +2041,7 @@ def test_legacy_converters(self): block = self.parse('module os\nos.access\n path: "s"') module, function = block.signatures conv = (function.parameters['path']).converter - self.assertIsInstance(conv, clinic.str_converter) + self.assertIsInstance(conv, str_converter) def test_legacy_converters_non_string_constant_annotation(self): err = "Annotations must be either a name, a function call, or a string" diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index ea480e61ba9a2b..a4e004d5b124d1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -8,7 +8,6 @@ import argparse import ast -import builtins as bltns import contextlib import dataclasses as dc import enum @@ -46,7 +45,7 @@ import libclinic.cpp from libclinic import ( ClinicError, Sentinels, VersionTuple, - fail, warn, unspecified, unknown) + fail, warn, unspecified, unknown, NULL) from libclinic.function import ( Module, Class, Function, Parameter, ClassDict, ModuleDict, FunctionKind, @@ -56,8 +55,11 @@ from libclinic.block_parser import Block, BlockParser from libclinic.crenderdata import CRenderData, Include, TemplateDict from libclinic.converter import ( - CConverter, CConverterClassT, ConverterType, + CConverter, ConverterType, converters, legacy_converters) +from libclinic.converters import ( + self_converter, defining_class_converter, object_converter, buffer, + robuffer, rwbuffer, correct_name_for_self) # TODO: @@ -77,15 +79,6 @@ LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API') -# This one needs to be a distinct class, unlike the other two -class Null: - def __repr__(self) -> str: - return '' - - -NULL = Null() - - ParamTuple = tuple["Parameter", ...] @@ -2098,24 +2091,6 @@ def parse(self, block: Block) -> None: ReturnConverterType = Callable[..., "CReturnConverter"] -def add_legacy_c_converter( - format_unit: str, - **kwargs: Any -) -> Callable[[CConverterClassT], CConverterClassT]: - """ - Adds a legacy converter. - """ - def closure(f: CConverterClassT) -> CConverterClassT: - added_f: Callable[..., CConverter] - if not kwargs: - added_f = f - else: - added_f = functools.partial(f, **kwargs) - if format_unit: - legacy_converters[format_unit] = added_f - return f - return closure - # maps strings to callables. # these callables must be of the form: # def foo(*, ...) @@ -2125,1179 +2100,6 @@ def closure(f: CConverterClassT) -> CConverterClassT: ReturnConverterDict = dict[str, ReturnConverterType] return_converters: ReturnConverterDict = {} -TypeSet = set[bltns.type[object]] - - -class bool_converter(CConverter): - type = 'int' - default_type = bool - format_unit = 'p' - c_ignored_default = '0' - - def converter_init(self, *, accept: TypeSet = {object}) -> None: - if accept == {int}: - self.format_unit = 'i' - elif accept != {object}: - fail(f"bool_converter: illegal 'accept' argument {accept!r}") - if self.default is not unspecified and self.default is not unknown: - self.default = bool(self.default) - self.c_default = str(int(self.default)) - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'i': - return self.format_code(""" - {paramname} = PyLong_AsInt({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - elif self.format_unit == 'p': - return self.format_code(""" - {paramname} = PyObject_IsTrue({argname}); - if ({paramname} < 0) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class defining_class_converter(CConverter): - """ - A special-case converter: - this is the default converter used for the defining class. - """ - type = 'PyTypeObject *' - format_unit = '' - show_in_signature = False - - def converter_init(self, *, type: str | None = None) -> None: - self.specified_type = type - - def render(self, parameter: Parameter, data: CRenderData) -> None: - self._render_self(parameter, data) - - def set_template_dict(self, template_dict: TemplateDict) -> None: - template_dict['defining_class_name'] = self.name - - -class char_converter(CConverter): - type = 'char' - default_type = (bytes, bytearray) - format_unit = 'c' - c_ignored_default = "'\0'" - - def converter_init(self) -> None: - if isinstance(self.default, self.default_type): - if len(self.default) != 1: - fail(f"char_converter: illegal default value {self.default!r}") - - self.c_default = repr(bytes(self.default))[1:] - if self.c_default == '"\'"': - self.c_default = r"'\''" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'c': - return self.format_code(""" - if (PyBytes_Check({argname}) && PyBytes_GET_SIZE({argname}) == 1) {{{{ - {paramname} = PyBytes_AS_STRING({argname})[0]; - }}}} - else if (PyByteArray_Check({argname}) && PyByteArray_GET_SIZE({argname}) == 1) {{{{ - {paramname} = PyByteArray_AS_STRING({argname})[0]; - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'a byte string of length 1', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -@add_legacy_c_converter('B', bitwise=True) -class unsigned_char_converter(CConverter): - type = 'unsigned char' - default_type = int - format_unit = 'b' - c_ignored_default = "'\0'" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'B' - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'b': - return self.format_code(""" - {{{{ - long ival = PyLong_AsLong({argname}); - if (ival == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else if (ival < 0) {{{{ - PyErr_SetString(PyExc_OverflowError, - "unsigned byte integer is less than minimum"); - goto exit; - }}}} - else if (ival > UCHAR_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "unsigned byte integer is greater than maximum"); - goto exit; - }}}} - else {{{{ - {paramname} = (unsigned char) ival; - }}}} - }}}} - """, - argname=argname) - elif self.format_unit == 'B': - return self.format_code(""" - {{{{ - unsigned long ival = PyLong_AsUnsignedLongMask({argname}); - if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else {{{{ - {paramname} = (unsigned char) ival; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class byte_converter(unsigned_char_converter): pass - -class short_converter(CConverter): - type = 'short' - default_type = int - format_unit = 'h' - c_ignored_default = "0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'h': - return self.format_code(""" - {{{{ - long ival = PyLong_AsLong({argname}); - if (ival == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else if (ival < SHRT_MIN) {{{{ - PyErr_SetString(PyExc_OverflowError, - "signed short integer is less than minimum"); - goto exit; - }}}} - else if (ival > SHRT_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "signed short integer is greater than maximum"); - goto exit; - }}}} - else {{{{ - {paramname} = (short) ival; - }}}} - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_short_converter(CConverter): - type = 'unsigned short' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'H' - else: - self.converter = '_PyLong_UnsignedShort_Converter' - - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedShort_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedShort_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'H': - return self.format_code(""" - {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {{{{ - unsigned long uval = PyLong_AsUnsignedLong({argname}); - if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - if (uval > USHRT_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "Python int too large for C unsigned short"); - goto exit; - }}}} - {paramname} = (unsigned short) uval; - }}}} - """, - argname=argname) - -@add_legacy_c_converter('C', accept={str}) -class int_converter(CConverter): - type = 'int' - default_type = int - format_unit = 'i' - c_ignored_default = "0" - - def converter_init( - self, *, accept: TypeSet = {int}, type: str | None = None - ) -> None: - if accept == {str}: - self.format_unit = 'C' - elif accept != {int}: - fail(f"int_converter: illegal 'accept' argument {accept!r}") - if type is not None: - self.type = type - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'i': - return self.format_code(""" - {paramname} = PyLong_AsInt({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - elif self.format_unit == 'C': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - if (PyUnicode_GET_LENGTH({argname}) != 1) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyUnicode_READ_CHAR({argname}, 0); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'a unicode character', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_int_converter(CConverter): - type = 'unsigned int' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'I' - else: - self.converter = '_PyLong_UnsignedInt_Converter' - - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedInt_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedInt_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'I': - return self.format_code(""" - {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {{{{ - unsigned long uval = PyLong_AsUnsignedLong({argname}); - if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - if (uval > UINT_MAX) {{{{ - PyErr_SetString(PyExc_OverflowError, - "Python int too large for C unsigned int"); - goto exit; - }}}} - {paramname} = (unsigned int) uval; - }}}} - """, - argname=argname) - -class long_converter(CConverter): - type = 'long' - default_type = int - format_unit = 'l' - c_ignored_default = "0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'l': - return self.format_code(""" - {paramname} = PyLong_AsLong({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_long_converter(CConverter): - type = 'unsigned long' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'k' - else: - self.converter = '_PyLong_UnsignedLong_Converter' - - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedLong_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedLong_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'k': - return self.format_code(""" - if (!PyLong_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {paramname} = PyLong_AsUnsignedLong({argname}); - if ({paramname} == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - -class long_long_converter(CConverter): - type = 'long long' - default_type = int - format_unit = 'L' - c_ignored_default = "0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'L': - return self.format_code(""" - {paramname} = PyLong_AsLongLong({argname}); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unsigned_long_long_converter(CConverter): - type = 'unsigned long long' - default_type = int - c_ignored_default = "0" - - def converter_init(self, *, bitwise: bool = False) -> None: - if bitwise: - self.format_unit = 'K' - else: - self.converter = '_PyLong_UnsignedLongLong_Converter' - - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedLongLong_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedLongLong_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'K': - return self.format_code(""" - if (!PyLong_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {paramname} = PyLong_AsUnsignedLongLong({argname}); - if ({paramname} == (unsigned long long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - -class Py_ssize_t_converter(CConverter): - type = 'Py_ssize_t' - c_ignored_default = "0" - - def converter_init(self, *, accept: TypeSet = {int}) -> None: - if accept == {int}: - self.format_unit = 'n' - self.default_type = int - elif accept == {int, NoneType}: - self.converter = '_Py_convert_optional_to_ssize_t' - else: - fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") - - def use_converter(self) -> None: - if self.converter == '_Py_convert_optional_to_ssize_t': - self.add_include('pycore_abstract.h', - '_Py_convert_optional_to_ssize_t()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'n': - if limited_capi: - PyNumber_Index = 'PyNumber_Index' - else: - PyNumber_Index = '_PyNumber_Index' - self.add_include('pycore_abstract.h', '_PyNumber_Index()') - return self.format_code(""" - {{{{ - Py_ssize_t ival = -1; - PyObject *iobj = {PyNumber_Index}({argname}); - if (iobj != NULL) {{{{ - ival = PyLong_AsSsize_t(iobj); - Py_DECREF(iobj); - }}}} - if (ival == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - {paramname} = ival; - }}}} - """, - argname=argname, - PyNumber_Index=PyNumber_Index) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return self.format_code(""" - if ({argname} != Py_None) {{{{ - if (PyIndex_Check({argname})) {{{{ - {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'integer or None', limited_capi=limited_capi), - ) - - -class slice_index_converter(CConverter): - type = 'Py_ssize_t' - - def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: - if accept == {int}: - self.converter = '_PyEval_SliceIndexNotNone' - self.nullable = False - elif accept == {int, NoneType}: - self.converter = '_PyEval_SliceIndex' - self.nullable = True - else: - fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - if self.nullable: - return self.format_code(""" - if (!Py_IsNone({argname})) {{{{ - if (PyIndex_Check({argname})) {{{{ - {paramname} = PyNumber_AsSsize_t({argname}, NULL); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - return 0; - }}}} - }}}} - else {{{{ - PyErr_SetString(PyExc_TypeError, - "slice indices must be integers or " - "None or have an __index__ method"); - goto exit; - }}}} - }}}} - """, - argname=argname) - else: - return self.format_code(""" - if (PyIndex_Check({argname})) {{{{ - {paramname} = PyNumber_AsSsize_t({argname}, NULL); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - PyErr_SetString(PyExc_TypeError, - "slice indices must be integers or " - "have an __index__ method"); - goto exit; - }}}} - """, - argname=argname) - -class size_t_converter(CConverter): - type = 'size_t' - converter = '_PyLong_Size_t_Converter' - c_ignored_default = "0" - - def use_converter(self) -> None: - self.add_include('pycore_long.h', - '_PyLong_Size_t_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'n': - return self.format_code(""" - {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); - if ({paramname} == -1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - # NOTE: Raises OverflowError for negative integer. - return self.format_code(""" - {paramname} = PyLong_AsSize_t({argname}); - if ({paramname} == (size_t)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - - -class fildes_converter(CConverter): - type = 'int' - converter = '_PyLong_FileDescriptor_Converter' - - def use_converter(self) -> None: - self.add_include('pycore_fileutils.h', - '_PyLong_FileDescriptor_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - return self.format_code(""" - {paramname} = PyObject_AsFileDescriptor({argname}); - if ({paramname} < 0) {{{{ - goto exit; - }}}} - """, - argname=argname) - - -class float_converter(CConverter): - type = 'float' - default_type = float - format_unit = 'f' - c_ignored_default = "0.0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'f': - if not limited_capi: - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); - }}}} - else - {{{{ - {paramname} = (float) PyFloat_AsDouble({argname}); - if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - """, - argname=argname) - else: - return self.format_code(""" - {paramname} = (float) PyFloat_AsDouble({argname}); - if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class double_converter(CConverter): - type = 'double' - default_type = float - format_unit = 'd' - c_ignored_default = "0.0" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'd': - if not limited_capi: - return self.format_code(""" - if (PyFloat_CheckExact({argname})) {{{{ - {paramname} = PyFloat_AS_DOUBLE({argname}); - }}}} - else - {{{{ - {paramname} = PyFloat_AsDouble({argname}); - if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - }}}} - """, - argname=argname) - else: - return self.format_code(""" - {paramname} = PyFloat_AsDouble({argname}); - if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -class Py_complex_converter(CConverter): - type = 'Py_complex' - default_type = complex - format_unit = 'D' - c_ignored_default = "{0.0, 0.0}" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'D': - return self.format_code(""" - {paramname} = PyComplex_AsCComplex({argname}); - if (PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -class object_converter(CConverter): - type = 'PyObject *' - format_unit = 'O' - - def converter_init( - self, *, - converter: str | None = None, - type: str | None = None, - subclass_of: str | None = None - ) -> None: - if converter: - if subclass_of: - fail("object: Cannot pass in both 'converter' and 'subclass_of'") - self.format_unit = 'O&' - self.converter = converter - elif subclass_of: - self.format_unit = 'O!' - self.subclass_of = subclass_of - - if type is not None: - self.type = type - - -# -# We define three conventions for buffer types in the 'accept' argument: -# -# buffer : any object supporting the buffer interface -# rwbuffer: any object supporting the buffer interface, but must be writeable -# robuffer: any object supporting the buffer interface, but must not be writeable -# - -class buffer: pass -class rwbuffer: pass -class robuffer: pass - -StrConverterKeyType = tuple[frozenset[type[object]], bool, bool] - -def str_converter_key( - types: TypeSet, encoding: bool | str | None, zeroes: bool -) -> StrConverterKeyType: - return (frozenset(types), bool(encoding), bool(zeroes)) - -str_converter_argument_map: dict[StrConverterKeyType, str] = {} - -class str_converter(CConverter): - type = 'const char *' - default_type = (str, Null, NoneType) - format_unit = 's' - - def converter_init( - self, - *, - accept: TypeSet = {str}, - encoding: str | None = None, - zeroes: bool = False - ) -> None: - - key = str_converter_key(accept, encoding, zeroes) - format_unit = str_converter_argument_map.get(key) - if not format_unit: - fail("str_converter: illegal combination of arguments", key) - - self.format_unit = format_unit - self.length = bool(zeroes) - if encoding: - if self.default not in (Null, None, unspecified): - fail("str_converter: Argument Clinic doesn't support default values for encoded strings") - self.encoding = encoding - self.type = 'char *' - # sorry, clinic can't support preallocated buffers - # for es# and et# - self.c_default = "NULL" - if NoneType in accept and self.c_default == "Py_None": - self.c_default = "NULL" - - def post_parsing(self) -> str: - if self.encoding: - name = self.name - return f"PyMem_FREE({name});\n" - else: - return "" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 's': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - Py_ssize_t {length_name}; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - if (strlen({paramname}) != (size_t){length_name}) {{{{ - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - length_name=self.length_name) - if self.format_unit == 'z': - return self.format_code(""" - if ({argname} == Py_None) {{{{ - {paramname} = NULL; - }}}} - else if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t {length_name}; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - if (strlen({paramname}) != (size_t){length_name}) {{{{ - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - }}}} - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), - length_name=self.length_name) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -# -# This is the fourth or fifth rewrite of registering all the -# string converter format units. Previous approaches hid -# bugs--generally mismatches between the semantics of the format -# unit and the arguments necessary to represent those semantics -# properly. Hopefully with this approach we'll get it 100% right. -# -# The r() function (short for "register") both registers the -# mapping from arguments to format unit *and* registers the -# legacy C converter for that format unit. -# -def r(format_unit: str, - *, - accept: TypeSet, - encoding: bool = False, - zeroes: bool = False -) -> None: - if not encoding and format_unit != 's': - # add the legacy c converters here too. - # - # note: add_legacy_c_converter can't work for - # es, es#, et, or et# - # because of their extra encoding argument - # - # also don't add the converter for 's' because - # the metaclass for CConverter adds it for us. - kwargs: dict[str, Any] = {} - if accept != {str}: - kwargs['accept'] = accept - if zeroes: - kwargs['zeroes'] = True - added_f = functools.partial(str_converter, **kwargs) - legacy_converters[format_unit] = added_f - - d = str_converter_argument_map - key = str_converter_key(accept, encoding, zeroes) - if key in d: - sys.exit("Duplicate keys specified for str_converter_argument_map!") - d[key] = format_unit - -r('es', encoding=True, accept={str}) -r('es#', encoding=True, zeroes=True, accept={str}) -r('et', encoding=True, accept={bytes, bytearray, str}) -r('et#', encoding=True, zeroes=True, accept={bytes, bytearray, str}) -r('s', accept={str}) -r('s#', zeroes=True, accept={robuffer, str}) -r('y', accept={robuffer}) -r('y#', zeroes=True, accept={robuffer}) -r('z', accept={str, NoneType}) -r('z#', zeroes=True, accept={robuffer, str, NoneType}) -del r - - -class PyBytesObject_converter(CConverter): - type = 'PyBytesObject *' - format_unit = 'S' - # accept = {bytes} - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'S': - return self.format_code(""" - if (!PyBytes_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = ({type}){argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'bytes', limited_capi=limited_capi), - type=self.type) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class PyByteArrayObject_converter(CConverter): - type = 'PyByteArrayObject *' - format_unit = 'Y' - # accept = {bytearray} - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'Y': - return self.format_code(""" - if (!PyByteArray_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = ({type}){argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'bytearray', limited_capi=limited_capi), - type=self.type) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -class unicode_converter(CConverter): - type = 'PyObject *' - default_type = (str, Null, NoneType) - format_unit = 'U' - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'U': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = {argname}; - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -@add_legacy_c_converter('u') -@add_legacy_c_converter('u#', zeroes=True) -@add_legacy_c_converter('Z', accept={str, NoneType}) -@add_legacy_c_converter('Z#', accept={str, NoneType}, zeroes=True) -class Py_UNICODE_converter(CConverter): - type = 'const wchar_t *' - default_type = (str, Null, NoneType) - - def converter_init( - self, *, - accept: TypeSet = {str}, - zeroes: bool = False - ) -> None: - format_unit = 'Z' if accept=={str, NoneType} else 'u' - if zeroes: - format_unit += '#' - self.length = True - self.format_unit = format_unit - else: - self.accept = accept - if accept == {str}: - self.converter = '_PyUnicode_WideCharString_Converter' - elif accept == {str, NoneType}: - self.converter = '_PyUnicode_WideCharString_Opt_Converter' - else: - fail(f"Py_UNICODE_converter: illegal 'accept' argument {accept!r}") - self.c_default = "NULL" - - def cleanup(self) -> str: - if self.length: - return "" - else: - return f"""PyMem_Free((void *){self.parser_name});\n""" - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if not self.length: - if self.accept == {str}: - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyUnicode_AsWideCharString({argname}, NULL); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - ) - elif self.accept == {str, NoneType}: - return self.format_code(""" - if ({argname} == Py_None) {{{{ - {paramname} = NULL; - }}}} - else if (PyUnicode_Check({argname})) {{{{ - {paramname} = PyUnicode_AsWideCharString({argname}, NULL); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - -@add_legacy_c_converter('s*', accept={str, buffer}) -@add_legacy_c_converter('z*', accept={str, buffer, NoneType}) -@add_legacy_c_converter('w*', accept={rwbuffer}) -class Py_buffer_converter(CConverter): - type = 'Py_buffer' - format_unit = 'y*' - impl_by_reference = True - c_ignored_default = "{NULL, NULL}" - - def converter_init(self, *, accept: TypeSet = {buffer}) -> None: - if self.default not in (unspecified, None): - fail("The only legal default value for Py_buffer is None.") - - self.c_default = self.c_ignored_default - - if accept == {str, buffer, NoneType}: - format_unit = 'z*' - elif accept == {str, buffer}: - format_unit = 's*' - elif accept == {buffer}: - format_unit = 'y*' - elif accept == {rwbuffer}: - format_unit = 'w*' - else: - fail("Py_buffer_converter: illegal combination of arguments") - - self.format_unit = format_unit - - def cleanup(self) -> str: - name = self.name - return "".join(["if (", name, ".obj) {\n PyBuffer_Release(&", name, ");\n}\n"]) - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - # PyBUF_SIMPLE guarantees that the format units of the buffers are C-contiguous. - if self.format_unit == 'y*': - return self.format_code(""" - if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), - ) - elif self.format_unit == 's*': - return self.format_code(""" - if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t len; - const char *ptr = PyUnicode_AsUTF8AndSize({argname}, &len); - if (ptr == NULL) {{{{ - goto exit; - }}}} - if (PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) {{{{ - goto exit; - }}}} - }}}} - else {{{{ /* any bytes-like object */ - if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ - goto exit; - }}}} - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), - ) - elif self.format_unit == 'w*': - return self.format_code(""" - if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_WRITABLE) < 0) {{{{ - {bad_argument} - goto exit; - }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'read-write bytes-like object', limited_capi=limited_capi), - bad_argument2=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - - -def correct_name_for_self( - f: Function -) -> tuple[str, str]: - if f.kind in {CALLABLE, METHOD_INIT, GETTER, SETTER}: - if f.cls: - return "PyObject *", "self" - return "PyObject *", "module" - if f.kind is STATIC_METHOD: - return "void *", "null" - if f.kind in (CLASS_METHOD, METHOD_NEW): - return "PyTypeObject *", "type" - raise AssertionError(f"Unhandled type of function f: {f.kind!r}") - - -class self_converter(CConverter): - """ - A special-case converter: - this is the default converter used for "self". - """ - type: str | None = None - format_unit = '' - - def converter_init(self, *, type: str | None = None) -> None: - self.specified_type = type - - def pre_render(self) -> None: - f = self.function - default_type, default_name = correct_name_for_self(f) - self.signature_name = default_name - self.type = self.specified_type or self.type or default_type - - kind = self.function.kind - - if kind is STATIC_METHOD or kind.new_or_init: - self.show_in_signature = False - - # tp_new (METHOD_NEW) functions are of type newfunc: - # typedef PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *); - # - # tp_init (METHOD_INIT) functions are of type initproc: - # typedef int (*initproc)(PyObject *, PyObject *, PyObject *); - # - # All other functions generated by Argument Clinic are stored in - # PyMethodDef structures, in the ml_meth slot, which is of type PyCFunction: - # typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); - # However! We habitually cast these functions to PyCFunction, - # since functions that accept keyword arguments don't fit this signature - # but are stored there anyway. So strict type equality isn't important - # for these functions. - # - # So: - # - # * The name of the first parameter to the impl and the parsing function will always - # be self.name. - # - # * The type of the first parameter to the impl will always be of self.type. - # - # * If the function is neither tp_new (METHOD_NEW) nor tp_init (METHOD_INIT): - # * The type of the first parameter to the parsing function is also self.type. - # This means that if you step into the parsing function, your "self" parameter - # is of the correct type, which may make debugging more pleasant. - # - # * Else if the function is tp_new (METHOD_NEW): - # * The type of the first parameter to the parsing function is "PyTypeObject *", - # so the type signature of the function call is an exact match. - # * If self.type != "PyTypeObject *", we cast the first parameter to self.type - # in the impl call. - # - # * Else if the function is tp_init (METHOD_INIT): - # * The type of the first parameter to the parsing function is "PyObject *", - # so the type signature of the function call is an exact match. - # * If self.type != "PyObject *", we cast the first parameter to self.type - # in the impl call. - - @property - def parser_type(self) -> str: - assert self.type is not None - if self.function.kind in {METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD}: - tp, _ = correct_name_for_self(self.function) - return tp - return self.type - - def render(self, parameter: Parameter, data: CRenderData) -> None: - """ - parameter is a clinic.Parameter instance. - data is a CRenderData instance. - """ - if self.function.kind is STATIC_METHOD: - return - - self._render_self(parameter, data) - - if self.type != self.parser_type: - # insert cast to impl_argument[0], aka self. - # we know we're in the first slot in all the CRenderData lists, - # because we render parameters in order, and self is always first. - assert len(data.impl_arguments) == 1 - assert data.impl_arguments[0] == self.name - assert self.type is not None - data.impl_arguments[0] = '(' + self.type + ")" + data.impl_arguments[0] - - def set_template_dict(self, template_dict: TemplateDict) -> None: - template_dict['self_name'] = self.name - template_dict['self_type'] = self.parser_type - kind = self.function.kind - cls = self.function.cls - - if kind.new_or_init and cls and cls.typedef: - if kind is METHOD_NEW: - type_check = ( - '({0} == base_tp || {0}->tp_init == base_tp->tp_init)' - ).format(self.name) - else: - type_check = ('(Py_IS_TYPE({0}, base_tp) ||\n ' - ' Py_TYPE({0})->tp_new == base_tp->tp_new)' - ).format(self.name) - - line = f'{type_check} &&\n ' - template_dict['self_type_check'] = line - - type_object = cls.type_object - type_ptr = f'PyTypeObject *base_tp = {type_object};' - template_dict['base_type_ptr'] = type_ptr - def add_c_return_converter( f: ReturnConverterType, diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 32231b82bfdc07..7c5cede2396677 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -25,13 +25,15 @@ ) from .utils import ( FormatCounterFormatter, + NULL, + Null, + Sentinels, + VersionTuple, compute_checksum, create_regex, - write_file, - VersionTuple, - Sentinels, - unspecified, unknown, + unspecified, + write_file, ) @@ -61,13 +63,15 @@ # Utility functions "FormatCounterFormatter", + "NULL", + "Null", + "Sentinels", + "VersionTuple", "compute_checksum", "create_regex", - "write_file", - "VersionTuple", - "Sentinels", - "unspecified", "unknown", + "unspecified", + "write_file", ] diff --git a/Tools/clinic/libclinic/converter.py b/Tools/clinic/libclinic/converter.py index 744d03c2c1fea4..ac78be3f7958da 100644 --- a/Tools/clinic/libclinic/converter.py +++ b/Tools/clinic/libclinic/converter.py @@ -531,3 +531,19 @@ def add_include(self, name: str, reason: str, # these callables follow the same rules as those for "converters" above. # note however that they will never be called with keyword-only parameters. legacy_converters: ConverterDict = {} + + +def add_legacy_c_converter( + format_unit: str, + **kwargs: Any +) -> Callable[[CConverterClassT], CConverterClassT]: + def closure(f: CConverterClassT) -> CConverterClassT: + added_f: Callable[..., CConverter] + if not kwargs: + added_f = f + else: + added_f = functools.partial(f, **kwargs) + if format_unit: + legacy_converters[format_unit] = added_f + return f + return closure diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py new file mode 100644 index 00000000000000..7fc16f17450aaa --- /dev/null +++ b/Tools/clinic/libclinic/converters.py @@ -0,0 +1,1211 @@ +import builtins as bltns +import functools +import sys +from types import NoneType +from typing import Any + +from libclinic import fail, Null, unspecified, unknown +from libclinic.function import ( + Function, Parameter, + CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, + GETTER, SETTER) +from libclinic.crenderdata import CRenderData, TemplateDict +from libclinic.converter import ( + CConverter, legacy_converters, add_legacy_c_converter) + + +TypeSet = set[bltns.type[object]] + + +class bool_converter(CConverter): + type = 'int' + default_type = bool + format_unit = 'p' + c_ignored_default = '0' + + def converter_init(self, *, accept: TypeSet = {object}) -> None: + if accept == {int}: + self.format_unit = 'i' + elif accept != {object}: + fail(f"bool_converter: illegal 'accept' argument {accept!r}") + if self.default is not unspecified and self.default is not unknown: + self.default = bool(self.default) + self.c_default = str(int(self.default)) + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'i': + return self.format_code(""" + {paramname} = PyLong_AsInt({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + elif self.format_unit == 'p': + return self.format_code(""" + {paramname} = PyObject_IsTrue({argname}); + if ({paramname} < 0) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class defining_class_converter(CConverter): + """ + A special-case converter: + this is the default converter used for the defining class. + """ + type = 'PyTypeObject *' + format_unit = '' + show_in_signature = False + + def converter_init(self, *, type: str | None = None) -> None: + self.specified_type = type + + def render(self, parameter: Parameter, data: CRenderData) -> None: + self._render_self(parameter, data) + + def set_template_dict(self, template_dict: TemplateDict) -> None: + template_dict['defining_class_name'] = self.name + + +class char_converter(CConverter): + type = 'char' + default_type = (bytes, bytearray) + format_unit = 'c' + c_ignored_default = "'\0'" + + def converter_init(self) -> None: + if isinstance(self.default, self.default_type): + if len(self.default) != 1: + fail(f"char_converter: illegal default value {self.default!r}") + + self.c_default = repr(bytes(self.default))[1:] + if self.c_default == '"\'"': + self.c_default = r"'\''" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'c': + return self.format_code(""" + if (PyBytes_Check({argname}) && PyBytes_GET_SIZE({argname}) == 1) {{{{ + {paramname} = PyBytes_AS_STRING({argname})[0]; + }}}} + else if (PyByteArray_Check({argname}) && PyByteArray_GET_SIZE({argname}) == 1) {{{{ + {paramname} = PyByteArray_AS_STRING({argname})[0]; + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'a byte string of length 1', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +@add_legacy_c_converter('B', bitwise=True) +class unsigned_char_converter(CConverter): + type = 'unsigned char' + default_type = int + format_unit = 'b' + c_ignored_default = "'\0'" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'B' + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'b': + return self.format_code(""" + {{{{ + long ival = PyLong_AsLong({argname}); + if (ival == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + else if (ival < 0) {{{{ + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + }}}} + else if (ival > UCHAR_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + }}}} + else {{{{ + {paramname} = (unsigned char) ival; + }}}} + }}}} + """, + argname=argname) + elif self.format_unit == 'B': + return self.format_code(""" + {{{{ + unsigned long ival = PyLong_AsUnsignedLongMask({argname}); + if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + else {{{{ + {paramname} = (unsigned char) ival; + }}}} + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class byte_converter(unsigned_char_converter): + pass + + +class short_converter(CConverter): + type = 'short' + default_type = int + format_unit = 'h' + c_ignored_default = "0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'h': + return self.format_code(""" + {{{{ + long ival = PyLong_AsLong({argname}); + if (ival == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + else if (ival < SHRT_MIN) {{{{ + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + }}}} + else if (ival > SHRT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + }}}} + else {{{{ + {paramname} = (short) ival; + }}}} + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_short_converter(CConverter): + type = 'unsigned short' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'H' + else: + self.converter = '_PyLong_UnsignedShort_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedShort_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedShort_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'H': + return self.format_code(""" + {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); + if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {{{{ + unsigned long uval = PyLong_AsUnsignedLong({argname}); + if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + if (uval > USHRT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C unsigned short"); + goto exit; + }}}} + {paramname} = (unsigned short) uval; + }}}} + """, + argname=argname) + + +@add_legacy_c_converter('C', accept={str}) +class int_converter(CConverter): + type = 'int' + default_type = int + format_unit = 'i' + c_ignored_default = "0" + + def converter_init( + self, *, accept: TypeSet = {int}, type: str | None = None + ) -> None: + if accept == {str}: + self.format_unit = 'C' + elif accept != {int}: + fail(f"int_converter: illegal 'accept' argument {accept!r}") + if type is not None: + self.type = type + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'i': + return self.format_code(""" + {paramname} = PyLong_AsInt({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + elif self.format_unit == 'C': + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + if (PyUnicode_GET_LENGTH({argname}) != 1) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyUnicode_READ_CHAR({argname}, 0); + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'a unicode character', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_int_converter(CConverter): + type = 'unsigned int' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'I' + else: + self.converter = '_PyLong_UnsignedInt_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedInt_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedInt_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'I': + return self.format_code(""" + {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); + if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {{{{ + unsigned long uval = PyLong_AsUnsignedLong({argname}); + if (uval == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + if (uval > UINT_MAX) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C unsigned int"); + goto exit; + }}}} + {paramname} = (unsigned int) uval; + }}}} + """, + argname=argname) + + +class long_converter(CConverter): + type = 'long' + default_type = int + format_unit = 'l' + c_ignored_default = "0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'l': + return self.format_code(""" + {paramname} = PyLong_AsLong({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_long_converter(CConverter): + type = 'unsigned long' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'k' + else: + self.converter = '_PyLong_UnsignedLong_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedLong_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedLong_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'k': + return self.format_code(""" + if (!PyLong_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyLong_AsUnsignedLongMask({argname}); + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), + ) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsUnsignedLong({argname}); + if ({paramname} == (unsigned long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class long_long_converter(CConverter): + type = 'long long' + default_type = int + format_unit = 'L' + c_ignored_default = "0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'L': + return self.format_code(""" + {paramname} = PyLong_AsLongLong({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unsigned_long_long_converter(CConverter): + type = 'unsigned long long' + default_type = int + c_ignored_default = "0" + + def converter_init(self, *, bitwise: bool = False) -> None: + if bitwise: + self.format_unit = 'K' + else: + self.converter = '_PyLong_UnsignedLongLong_Converter' + + def use_converter(self) -> None: + if self.converter == '_PyLong_UnsignedLongLong_Converter': + self.add_include('pycore_long.h', + '_PyLong_UnsignedLongLong_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'K': + return self.format_code(""" + if (!PyLong_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyLong_AsUnsignedLongLongMask({argname}); + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), + ) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsUnsignedLongLong({argname}); + if ({paramname} == (unsigned long long)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class Py_ssize_t_converter(CConverter): + type = 'Py_ssize_t' + c_ignored_default = "0" + + def converter_init(self, *, accept: TypeSet = {int}) -> None: + if accept == {int}: + self.format_unit = 'n' + self.default_type = int + elif accept == {int, NoneType}: + self.converter = '_Py_convert_optional_to_ssize_t' + else: + fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") + + def use_converter(self) -> None: + if self.converter == '_Py_convert_optional_to_ssize_t': + self.add_include('pycore_abstract.h', + '_Py_convert_optional_to_ssize_t()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'n': + if limited_capi: + PyNumber_Index = 'PyNumber_Index' + else: + PyNumber_Index = '_PyNumber_Index' + self.add_include('pycore_abstract.h', '_PyNumber_Index()') + return self.format_code(""" + {{{{ + Py_ssize_t ival = -1; + PyObject *iobj = {PyNumber_Index}({argname}); + if (iobj != NULL) {{{{ + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + }}}} + if (ival == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + {paramname} = ival; + }}}} + """, + argname=argname, + PyNumber_Index=PyNumber_Index) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + return self.format_code(""" + if ({argname} != Py_None) {{{{ + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'integer or None', limited_capi=limited_capi), + ) + + +class slice_index_converter(CConverter): + type = 'Py_ssize_t' + + def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: + if accept == {int}: + self.converter = '_PyEval_SliceIndexNotNone' + self.nullable = False + elif accept == {int, NoneType}: + self.converter = '_PyEval_SliceIndex' + self.nullable = True + else: + fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + if self.nullable: + return self.format_code(""" + if (!Py_IsNone({argname})) {{{{ + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, NULL); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + return 0; + }}}} + }}}} + else {{{{ + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "None or have an __index__ method"); + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + if (PyIndex_Check({argname})) {{{{ + {paramname} = PyNumber_AsSsize_t({argname}, NULL); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + goto exit; + }}}} + """, + argname=argname) + + +class size_t_converter(CConverter): + type = 'size_t' + converter = '_PyLong_Size_t_Converter' + c_ignored_default = "0" + + def use_converter(self) -> None: + self.add_include('pycore_long.h', + '_PyLong_Size_t_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'n': + return self.format_code(""" + {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + # NOTE: Raises OverflowError for negative integer. + return self.format_code(""" + {paramname} = PyLong_AsSize_t({argname}); + if ({paramname} == (size_t)-1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class fildes_converter(CConverter): + type = 'int' + converter = '_PyLong_FileDescriptor_Converter' + + def use_converter(self) -> None: + self.add_include('pycore_fileutils.h', + '_PyLong_FileDescriptor_Converter()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + return self.format_code(""" + {paramname} = PyObject_AsFileDescriptor({argname}); + if ({paramname} < 0) {{{{ + goto exit; + }}}} + """, + argname=argname) + + +class float_converter(CConverter): + type = 'float' + default_type = float + format_unit = 'f' + c_ignored_default = "0.0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'f': + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); + }}}} + else + {{{{ + {paramname} = (float) PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + {paramname} = (float) PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class double_converter(CConverter): + type = 'double' + default_type = float + format_unit = 'd' + c_ignored_default = "0.0" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'd': + if not limited_capi: + return self.format_code(""" + if (PyFloat_CheckExact({argname})) {{{{ + {paramname} = PyFloat_AS_DOUBLE({argname}); + }}}} + else + {{{{ + {paramname} = PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname) + else: + return self.format_code(""" + {paramname} = PyFloat_AsDouble({argname}); + if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class Py_complex_converter(CConverter): + type = 'Py_complex' + default_type = complex + format_unit = 'D' + c_ignored_default = "{0.0, 0.0}" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'D': + return self.format_code(""" + {paramname} = PyComplex_AsCComplex({argname}); + if (PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, + argname=argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class object_converter(CConverter): + type = 'PyObject *' + format_unit = 'O' + + def converter_init( + self, *, + converter: str | None = None, + type: str | None = None, + subclass_of: str | None = None + ) -> None: + if converter: + if subclass_of: + fail("object: Cannot pass in both 'converter' and 'subclass_of'") + self.format_unit = 'O&' + self.converter = converter + elif subclass_of: + self.format_unit = 'O!' + self.subclass_of = subclass_of + + if type is not None: + self.type = type + + +# +# We define three conventions for buffer types in the 'accept' argument: +# +# buffer : any object supporting the buffer interface +# rwbuffer: any object supporting the buffer interface, but must be writeable +# robuffer: any object supporting the buffer interface, but must not be writeable +# + +class buffer: + pass +class rwbuffer: + pass +class robuffer: + pass + + +StrConverterKeyType = tuple[frozenset[type[object]], bool, bool] + +def str_converter_key( + types: TypeSet, encoding: bool | str | None, zeroes: bool +) -> StrConverterKeyType: + return (frozenset(types), bool(encoding), bool(zeroes)) + +str_converter_argument_map: dict[StrConverterKeyType, str] = {} + + +class str_converter(CConverter): + type = 'const char *' + default_type = (str, Null, NoneType) + format_unit = 's' + + def converter_init( + self, + *, + accept: TypeSet = {str}, + encoding: str | None = None, + zeroes: bool = False + ) -> None: + + key = str_converter_key(accept, encoding, zeroes) + format_unit = str_converter_argument_map.get(key) + if not format_unit: + fail("str_converter: illegal combination of arguments", key) + + self.format_unit = format_unit + self.length = bool(zeroes) + if encoding: + if self.default not in (Null, None, unspecified): + fail("str_converter: Argument Clinic doesn't support default values for encoded strings") + self.encoding = encoding + self.type = 'char *' + # sorry, clinic can't support preallocated buffers + # for es# and et# + self.c_default = "NULL" + if NoneType in accept and self.c_default == "Py_None": + self.c_default = "NULL" + + def post_parsing(self) -> str: + if self.encoding: + name = self.name + return f"PyMem_FREE({name});\n" + else: + return "" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 's': + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + if (strlen({paramname}) != (size_t){length_name}) {{{{ + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + length_name=self.length_name) + if self.format_unit == 'z': + return self.format_code(""" + if ({argname} == Py_None) {{{{ + {paramname} = NULL; + }}}} + else if (PyUnicode_Check({argname})) {{{{ + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + if (strlen({paramname}) != (size_t){length_name}) {{{{ + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), + length_name=self.length_name) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + +# +# This is the fourth or fifth rewrite of registering all the +# string converter format units. Previous approaches hid +# bugs--generally mismatches between the semantics of the format +# unit and the arguments necessary to represent those semantics +# properly. Hopefully with this approach we'll get it 100% right. +# +# The r() function (short for "register") both registers the +# mapping from arguments to format unit *and* registers the +# legacy C converter for that format unit. +# +def r(format_unit: str, + *, + accept: TypeSet, + encoding: bool = False, + zeroes: bool = False +) -> None: + if not encoding and format_unit != 's': + # add the legacy c converters here too. + # + # note: add_legacy_c_converter can't work for + # es, es#, et, or et# + # because of their extra encoding argument + # + # also don't add the converter for 's' because + # the metaclass for CConverter adds it for us. + kwargs: dict[str, Any] = {} + if accept != {str}: + kwargs['accept'] = accept + if zeroes: + kwargs['zeroes'] = True + added_f = functools.partial(str_converter, **kwargs) + legacy_converters[format_unit] = added_f + + d = str_converter_argument_map + key = str_converter_key(accept, encoding, zeroes) + if key in d: + sys.exit("Duplicate keys specified for str_converter_argument_map!") + d[key] = format_unit + +r('es', encoding=True, accept={str}) +r('es#', encoding=True, zeroes=True, accept={str}) +r('et', encoding=True, accept={bytes, bytearray, str}) +r('et#', encoding=True, zeroes=True, accept={bytes, bytearray, str}) +r('s', accept={str}) +r('s#', zeroes=True, accept={robuffer, str}) +r('y', accept={robuffer}) +r('y#', zeroes=True, accept={robuffer}) +r('z', accept={str, NoneType}) +r('z#', zeroes=True, accept={robuffer, str, NoneType}) +del r + + +class PyBytesObject_converter(CConverter): + type = 'PyBytesObject *' + format_unit = 'S' + # accept = {bytes} + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'S': + return self.format_code(""" + if (!PyBytes_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = ({type}){argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'bytes', limited_capi=limited_capi), + type=self.type) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class PyByteArrayObject_converter(CConverter): + type = 'PyByteArrayObject *' + format_unit = 'Y' + # accept = {bytearray} + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'Y': + return self.format_code(""" + if (!PyByteArray_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = ({type}){argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'bytearray', limited_capi=limited_capi), + type=self.type) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +class unicode_converter(CConverter): + type = 'PyObject *' + default_type = (str, Null, NoneType) + format_unit = 'U' + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.format_unit == 'U': + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = {argname}; + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +@add_legacy_c_converter('u') +@add_legacy_c_converter('u#', zeroes=True) +@add_legacy_c_converter('Z', accept={str, NoneType}) +@add_legacy_c_converter('Z#', accept={str, NoneType}, zeroes=True) +class Py_UNICODE_converter(CConverter): + type = 'const wchar_t *' + default_type = (str, Null, NoneType) + + def converter_init( + self, *, + accept: TypeSet = {str}, + zeroes: bool = False + ) -> None: + format_unit = 'Z' if accept=={str, NoneType} else 'u' + if zeroes: + format_unit += '#' + self.length = True + self.format_unit = format_unit + else: + self.accept = accept + if accept == {str}: + self.converter = '_PyUnicode_WideCharString_Converter' + elif accept == {str, NoneType}: + self.converter = '_PyUnicode_WideCharString_Opt_Converter' + else: + fail(f"Py_UNICODE_converter: illegal 'accept' argument {accept!r}") + self.c_default = "NULL" + + def cleanup(self) -> str: + if self.length: + return "" + else: + return f"""PyMem_Free((void *){self.parser_name});\n""" + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if not self.length: + if self.accept == {str}: + return self.format_code(""" + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {paramname} = PyUnicode_AsWideCharString({argname}, NULL); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), + ) + elif self.accept == {str, NoneType}: + return self.format_code(""" + if ({argname} == Py_None) {{{{ + {paramname} = NULL; + }}}} + else if (PyUnicode_Check({argname})) {{{{ + {paramname} = PyUnicode_AsWideCharString({argname}, NULL); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +@add_legacy_c_converter('s*', accept={str, buffer}) +@add_legacy_c_converter('z*', accept={str, buffer, NoneType}) +@add_legacy_c_converter('w*', accept={rwbuffer}) +class Py_buffer_converter(CConverter): + type = 'Py_buffer' + format_unit = 'y*' + impl_by_reference = True + c_ignored_default = "{NULL, NULL}" + + def converter_init(self, *, accept: TypeSet = {buffer}) -> None: + if self.default not in (unspecified, None): + fail("The only legal default value for Py_buffer is None.") + + self.c_default = self.c_ignored_default + + if accept == {str, buffer, NoneType}: + format_unit = 'z*' + elif accept == {str, buffer}: + format_unit = 's*' + elif accept == {buffer}: + format_unit = 'y*' + elif accept == {rwbuffer}: + format_unit = 'w*' + else: + fail("Py_buffer_converter: illegal combination of arguments") + + self.format_unit = format_unit + + def cleanup(self) -> str: + name = self.name + return "".join(["if (", name, ".obj) {\n PyBuffer_Release(&", name, ");\n}\n"]) + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + # PyBUF_SIMPLE guarantees that the format units of the buffers are C-contiguous. + if self.format_unit == 'y*': + return self.format_code(""" + if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + elif self.format_unit == 's*': + return self.format_code(""" + if (PyUnicode_Check({argname})) {{{{ + Py_ssize_t len; + const char *ptr = PyUnicode_AsUTF8AndSize({argname}, &len); + if (ptr == NULL) {{{{ + goto exit; + }}}} + if (PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, PyBUF_SIMPLE) < 0) {{{{ + goto exit; + }}}} + }}}} + else {{{{ /* any bytes-like object */ + if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ + goto exit; + }}}} + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + elif self.format_unit == 'w*': + return self.format_code(""" + if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_WRITABLE) < 0) {{{{ + {bad_argument} + goto exit; + }}}} + """, + argname=argname, + bad_argument=self.bad_argument(displayname, 'read-write bytes-like object', limited_capi=limited_capi), + bad_argument2=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), + ) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + +def correct_name_for_self( + f: Function +) -> tuple[str, str]: + if f.kind in {CALLABLE, METHOD_INIT, GETTER, SETTER}: + if f.cls: + return "PyObject *", "self" + return "PyObject *", "module" + if f.kind is STATIC_METHOD: + return "void *", "null" + if f.kind in (CLASS_METHOD, METHOD_NEW): + return "PyTypeObject *", "type" + raise AssertionError(f"Unhandled type of function f: {f.kind!r}") + + +class self_converter(CConverter): + """ + A special-case converter: + this is the default converter used for "self". + """ + type: str | None = None + format_unit = '' + + def converter_init(self, *, type: str | None = None) -> None: + self.specified_type = type + + def pre_render(self) -> None: + f = self.function + default_type, default_name = correct_name_for_self(f) + self.signature_name = default_name + self.type = self.specified_type or self.type or default_type + + kind = self.function.kind + + if kind is STATIC_METHOD or kind.new_or_init: + self.show_in_signature = False + + # tp_new (METHOD_NEW) functions are of type newfunc: + # typedef PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *); + # + # tp_init (METHOD_INIT) functions are of type initproc: + # typedef int (*initproc)(PyObject *, PyObject *, PyObject *); + # + # All other functions generated by Argument Clinic are stored in + # PyMethodDef structures, in the ml_meth slot, which is of type PyCFunction: + # typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); + # However! We habitually cast these functions to PyCFunction, + # since functions that accept keyword arguments don't fit this signature + # but are stored there anyway. So strict type equality isn't important + # for these functions. + # + # So: + # + # * The name of the first parameter to the impl and the parsing function will always + # be self.name. + # + # * The type of the first parameter to the impl will always be of self.type. + # + # * If the function is neither tp_new (METHOD_NEW) nor tp_init (METHOD_INIT): + # * The type of the first parameter to the parsing function is also self.type. + # This means that if you step into the parsing function, your "self" parameter + # is of the correct type, which may make debugging more pleasant. + # + # * Else if the function is tp_new (METHOD_NEW): + # * The type of the first parameter to the parsing function is "PyTypeObject *", + # so the type signature of the function call is an exact match. + # * If self.type != "PyTypeObject *", we cast the first parameter to self.type + # in the impl call. + # + # * Else if the function is tp_init (METHOD_INIT): + # * The type of the first parameter to the parsing function is "PyObject *", + # so the type signature of the function call is an exact match. + # * If self.type != "PyObject *", we cast the first parameter to self.type + # in the impl call. + + @property + def parser_type(self) -> str: + assert self.type is not None + if self.function.kind in {METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD}: + tp, _ = correct_name_for_self(self.function) + return tp + return self.type + + def render(self, parameter: Parameter, data: CRenderData) -> None: + """ + parameter is a clinic.Parameter instance. + data is a CRenderData instance. + """ + if self.function.kind is STATIC_METHOD: + return + + self._render_self(parameter, data) + + if self.type != self.parser_type: + # insert cast to impl_argument[0], aka self. + # we know we're in the first slot in all the CRenderData lists, + # because we render parameters in order, and self is always first. + assert len(data.impl_arguments) == 1 + assert data.impl_arguments[0] == self.name + assert self.type is not None + data.impl_arguments[0] = '(' + self.type + ")" + data.impl_arguments[0] + + def set_template_dict(self, template_dict: TemplateDict) -> None: + template_dict['self_name'] = self.name + template_dict['self_type'] = self.parser_type + kind = self.function.kind + cls = self.function.cls + + if kind.new_or_init and cls and cls.typedef: + if kind is METHOD_NEW: + type_check = ( + '({0} == base_tp || {0}->tp_init == base_tp->tp_init)' + ).format(self.name) + else: + type_check = ('(Py_IS_TYPE({0}, base_tp) ||\n ' + ' Py_TYPE({0})->tp_new == base_tp->tp_new)' + ).format(self.name) + + line = f'{type_check} &&\n ' + template_dict['self_type_check'] = line + + type_object = cls.type_object + type_ptr = f'PyTypeObject *base_tp = {type_object};' + template_dict['base_type_ptr'] = type_ptr diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index 4fafedb617115c..b0dd08446e802d 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -6,8 +6,9 @@ import inspect from typing import Final, Any, TYPE_CHECKING if TYPE_CHECKING: - from clinic import Clinic, CReturnConverter, self_converter + from clinic import Clinic, CReturnConverter from libclinic.converter import CConverter + from libclinic.converters import self_converter from libclinic import VersionTuple, unspecified diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py index 95a69f70c5499d..17e8f35be73bf4 100644 --- a/Tools/clinic/libclinic/utils.py +++ b/Tools/clinic/libclinic/utils.py @@ -82,3 +82,12 @@ def __repr__(self) -> str: unspecified: Final = Sentinels.unspecified unknown: Final = Sentinels.unknown + + +# This one needs to be a distinct class, unlike the other two +class Null: + def __repr__(self) -> str: + return '' + + +NULL = Null() From 1d5479b236e9a66dd32a24eff6fb83e3242b999d Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 2 Apr 2024 11:34:49 +0100 Subject: [PATCH 044/143] gh-117411: move PyFutureFeatures to pycore_symtable.h and make it private (#117412) --- Include/cpython/compile.h | 20 ------------- Include/internal/pycore_compile.h | 8 +++-- Include/internal/pycore_flowgraph.h | 2 +- Include/internal/pycore_symtable.h | 29 +++++++++++++++++-- ...-04-02-10-04-57.gh-issue-117411.YdyVmG.rst | 1 + Python/assemble.c | 3 +- Python/compile.c | 8 ++--- Python/flowgraph.c | 6 ++-- Python/future.c | 9 +++--- Python/symtable.c | 4 +-- Python/traceback.c | 1 - 11 files changed, 49 insertions(+), 42 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-02-10-04-57.gh-issue-117411.YdyVmG.rst diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index 0d587505ef7f85..cfdb7080d45f2b 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -32,28 +32,8 @@ typedef struct { #define _PyCompilerFlags_INIT \ (PyCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION} -/* source location information */ -typedef struct { - int lineno; - int end_lineno; - int col_offset; - int end_col_offset; -} _PyCompilerSrcLocation; - -#define SRC_LOCATION_FROM_AST(n) \ - (_PyCompilerSrcLocation){ \ - .lineno = (n)->lineno, \ - .end_lineno = (n)->end_lineno, \ - .col_offset = (n)->col_offset, \ - .end_col_offset = (n)->end_col_offset } - /* Future feature support */ -typedef struct { - int ff_features; /* flags set by future statements */ - _PyCompilerSrcLocation ff_location; /* location of last future statement */ -} PyFutureFeatures; - #define FUTURE_NESTED_SCOPES "nested_scopes" #define FUTURE_GENERATORS "generators" #define FUTURE_DIVISION "division" diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index f54f4f7f37acee..eb6e5ca58f7390 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_symtable.h" // _Py_SourceLocation + struct _arena; // Type defined in pycore_pyarena.h struct _mod; // Type defined in pycore_ast.h @@ -27,7 +29,7 @@ extern int _PyCompile_AstOptimize( int optimize, struct _arena *arena); -static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; +struct _Py_SourceLocation; extern int _PyAST_Optimize( struct _mod *, @@ -44,7 +46,7 @@ typedef struct { typedef struct { int i_opcode; int i_oparg; - _PyCompilerSrcLocation i_loc; + _Py_SourceLocation i_loc; _PyCompile_ExceptHandlerInfo i_except_handler_info; /* Used by the assembler */ @@ -65,7 +67,7 @@ typedef struct { int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl); int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq, int opcode, int oparg, - _PyCompilerSrcLocation loc); + _Py_SourceLocation loc); int _PyCompile_InstructionSequence_ApplyLabelMap(_PyCompile_InstructionSequence *seq); typedef struct { diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 58fed46886ea45..121302aacb3a8b 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -18,7 +18,7 @@ typedef struct { struct _PyCfgBuilder; int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); -int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); +int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _Py_SourceLocation loc); struct _PyCfgBuilder* _PyCfgBuilder_New(void); void _PyCfgBuilder_Free(struct _PyCfgBuilder *g); diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index b44393b5644673..16e89f80d9d0c8 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -29,6 +29,29 @@ typedef enum _comprehension_type { SetComprehension = 3, GeneratorExpression = 4 } _Py_comprehension_ty; +/* source location information */ +typedef struct { + int lineno; + int end_lineno; + int col_offset; + int end_col_offset; +} _Py_SourceLocation; + +#define SRC_LOCATION_FROM_AST(n) \ + (_Py_SourceLocation){ \ + .lineno = (n)->lineno, \ + .end_lineno = (n)->end_lineno, \ + .col_offset = (n)->col_offset, \ + .end_col_offset = (n)->end_col_offset } + +static const _Py_SourceLocation NO_LOCATION = {-1, -1, -1, -1}; + +/* __future__ information */ +typedef struct { + int ff_features; /* flags set by future statements */ + _Py_SourceLocation ff_location; /* location of last future statement */ +} _PyFutureFeatures; + struct _symtable_entry; struct symtable { @@ -44,7 +67,7 @@ struct symtable { consistency with the corresponding compiler structure */ PyObject *st_private; /* name of current class or NULL */ - PyFutureFeatures *st_future; /* module's future features that affect + _PyFutureFeatures *st_future; /* module's future features that affect the symbol table */ int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ @@ -100,7 +123,7 @@ extern int _PyST_IsFunctionLike(PySTEntryObject *); extern struct symtable* _PySymtable_Build( struct _mod *mod, PyObject *filename, - PyFutureFeatures *future); + _PyFutureFeatures *future); extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); extern void _PySymtable_Free(struct symtable *); @@ -150,7 +173,7 @@ extern struct symtable* _Py_SymtableStringObjectFlags( int _PyFuture_FromAST( struct _mod * mod, PyObject *filename, - PyFutureFeatures* futures); + _PyFutureFeatures* futures); #ifdef __cplusplus } diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-10-04-57.gh-issue-117411.YdyVmG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-10-04-57.gh-issue-117411.YdyVmG.rst new file mode 100644 index 00000000000000..73c60ee33a5413 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-10-04-57.gh-issue-117411.YdyVmG.rst @@ -0,0 +1 @@ +Move ``PyFutureFeatures`` to an internal header and make it private. diff --git a/Python/assemble.c b/Python/assemble.c index 09db2fab48d95c..be3d9c1a74657c 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -5,6 +5,7 @@ #include "pycore_compile.h" #include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE #include "pycore_opcode_metadata.h" // is_pseudo_target, _PyOpcode_Caches +#include "pycore_symtable.h" // _Py_SourceLocation #define DEFAULT_CODE_SIZE 128 @@ -21,7 +22,7 @@ return ERROR; \ } -typedef _PyCompilerSrcLocation location; +typedef _Py_SourceLocation location; typedef _PyCompile_Instruction instruction; typedef _PyCompile_InstructionSequence instr_sequence; diff --git a/Python/compile.c b/Python/compile.c index 43b3cbd4e1894c..d9312f93d0680f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -70,11 +70,11 @@ ((C)->c_flags.cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \ && ((C)->u->u_ste->ste_type == ModuleBlock)) -typedef _PyCompilerSrcLocation location; +typedef _Py_SourceLocation location; typedef struct _PyCfgBuilder cfg_builder; #define LOCATION(LNO, END_LNO, COL, END_COL) \ - ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) + ((const _Py_SourceLocation){(LNO), (END_LNO), (COL), (END_COL)}) /* Return true if loc1 starts after loc2 ends. */ static inline bool @@ -408,7 +408,7 @@ handled by the symbol analysis pass. struct compiler { PyObject *c_filename; struct symtable *c_st; - PyFutureFeatures c_future; /* module's __future__ */ + _PyFutureFeatures c_future; /* module's __future__ */ PyCompilerFlags c_flags; int c_optimize; /* optimization level */ @@ -585,7 +585,7 @@ int _PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, int optimize, PyArena *arena) { - PyFutureFeatures future; + _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { return -1; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 5437c3875ff7b0..9d98f6910cdf54 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -22,13 +22,13 @@ #define DEFAULT_BLOCK_SIZE 16 -typedef _PyCompilerSrcLocation location; +typedef _Py_SourceLocation location; typedef _PyCfgJumpTargetLabel jump_target_label; typedef struct _PyCfgInstruction { int i_opcode; int i_oparg; - _PyCompilerSrcLocation i_loc; + _Py_SourceLocation i_loc; struct _PyCfgBasicblock *i_target; /* target block (if jump instruction) */ struct _PyCfgBasicblock *i_except; /* target block when exception is raised */ } cfg_instr; @@ -92,7 +92,7 @@ static const jump_target_label NO_LABEL = {-1}; #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) #define LOCATION(LNO, END_LNO, COL, END_COL) \ - ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) + ((const _Py_SourceLocation){(LNO), (END_LNO), (COL), (END_COL)}) static inline int is_block_push(cfg_instr *i) diff --git a/Python/future.c b/Python/future.c index 0dbc7ede20f324..399345bd8fcbd9 100644 --- a/Python/future.c +++ b/Python/future.c @@ -1,11 +1,12 @@ #include "Python.h" #include "pycore_ast.h" // _PyAST_GetDocString() +#include "pycore_symtable.h" // _PyFutureFeatures #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined" static int -future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename) +future_check_features(_PyFutureFeatures *ff, stmt_ty s, PyObject *filename) { int i; @@ -53,7 +54,7 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename) } static int -future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) +future_parse(_PyFutureFeatures *ff, mod_ty mod, PyObject *filename) { if (!(mod->kind == Module_kind || mod->kind == Interactive_kind)) { return 1; @@ -98,10 +99,10 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) int -_PyFuture_FromAST(mod_ty mod, PyObject *filename, PyFutureFeatures *ff) +_PyFuture_FromAST(mod_ty mod, PyObject *filename, _PyFutureFeatures *ff) { ff->ff_features = 0; - ff->ff_location = (_PyCompilerSrcLocation){-1, -1, -1, -1}; + ff->ff_location = (_Py_SourceLocation){-1, -1, -1, -1}; if (!future_parse(ff, mod, filename)) { return 0; diff --git a/Python/symtable.c b/Python/symtable.c index b69452bf77c517..36ccc0e73723d5 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -387,7 +387,7 @@ symtable_new(void) } struct symtable * -_PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) +_PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) { struct symtable *st = symtable_new(); asdl_stmt_seq *seq; @@ -2757,7 +2757,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, _PyArena_Free(arena); return NULL; } - PyFutureFeatures future; + _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { _PyArena_Free(arena); return NULL; diff --git a/Python/traceback.c b/Python/traceback.c index 7a188e56c939c0..2564a7db5dcfec 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -5,7 +5,6 @@ #include "pycore_ast.h" // asdl_seq_GET() #include "pycore_call.h" // _PyObject_CallMethodFormat() -#include "pycore_compile.h" // _PyAST_Optimize() #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_frame.h" // _PyFrame_GetCode() #include "pycore_interp.h" // PyInterpreterState.gc From c97d3af2391e62ef456ef2365d48ab9b8cdbe27b Mon Sep 17 00:00:00 2001 From: Grigoriev Semyon <33061489+grigoriev-semyon@users.noreply.github.com> Date: Tue, 2 Apr 2024 13:42:58 +0300 Subject: [PATCH 045/143] gh-109120: Fix syntax error in handlinh of incorrect star expressions (#117444) --- Grammar/python.gram | 6 +- Lib/test/test_syntax.py | 26 +- ...-04-02-06-16-49.gh-issue-109120.X485oN.rst | 2 + Parser/parser.c | 2647 +++++++++-------- 4 files changed, 1442 insertions(+), 1239 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-02-06-16-49.gh-issue-109120.X485oN.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 696649392ae45d..9564abf5ec314b 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1013,6 +1013,7 @@ kwargs[asdl_seq*]: starred_expression[expr_ty]: | invalid_starred_expression | '*' a=expression { _PyAST_Starred(a, Load, EXTRA) } + | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") } kwarg_or_starred[KeywordOrStarred*]: | invalid_kwarg @@ -1133,8 +1134,8 @@ func_type_comment[Token*]: # From here on, there are rules for invalid syntax with specialised error messages invalid_arguments: - | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' b='*' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "iterable argument unpacking follows keyword argument unpacking") } + | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) a=',' ','.(starred_expression !'=')+ { + RAISE_SYNTAX_ERROR_STARTING_FROM(a, "iterable argument unpacking follows keyword argument unpacking") } | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") } | a=NAME b='=' expression for_if_clauses { @@ -1396,6 +1397,7 @@ invalid_kvpair: | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } invalid_starred_expression: | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") } + invalid_replacement_field: | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") } | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") } diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index d686dbf0c29149..dfa2a3b2f5413b 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1911,22 +1911,22 @@ >>> A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[*(1:2)] = 1 Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> del A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[*:] and A[:*] >>> A[*:] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[:*] Traceback (most recent call last): ... @@ -1937,7 +1937,7 @@ >>> A[*] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[**] @@ -2081,11 +2081,23 @@ def f(x: *b) >>> f(**x, *) Traceback (most recent call last): - SyntaxError: iterable argument unpacking follows keyword argument unpacking + SyntaxError: Invalid star expression >>> f(x, *:) Traceback (most recent call last): - SyntaxError: invalid syntax + SyntaxError: Invalid star expression + + >>> f(x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression """ import re diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-06-16-49.gh-issue-109120.X485oN.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-06-16-49.gh-issue-109120.X485oN.rst new file mode 100644 index 00000000000000..32e70b22f778e1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-06-16-49.gh-issue-109120.X485oN.rst @@ -0,0 +1,2 @@ +Added handle of incorrect star expressions, e.g ``f(3, *)``. Patch by +Grigoryev Semyon diff --git a/Parser/parser.c b/Parser/parser.c index 6817bd10d3cd7f..35d672b0d397f9 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -482,8 +482,8 @@ static char *soft_keywords[] = { #define _gather_148_type 1395 #define _tmp_150_type 1396 #define _tmp_151_type 1397 -#define _tmp_152_type 1398 -#define _tmp_153_type 1399 +#define _loop0_153_type 1398 +#define _gather_152_type 1399 #define _tmp_154_type 1400 #define _tmp_155_type 1401 #define _tmp_156_type 1402 @@ -492,50 +492,50 @@ static char *soft_keywords[] = { #define _tmp_159_type 1405 #define _tmp_160_type 1406 #define _tmp_161_type 1407 -#define _loop0_162_type 1408 -#define _loop0_163_type 1409 +#define _tmp_162_type 1408 +#define _tmp_163_type 1409 #define _loop0_164_type 1410 -#define _tmp_165_type 1411 -#define _tmp_166_type 1412 +#define _loop0_165_type 1411 +#define _loop0_166_type 1412 #define _tmp_167_type 1413 #define _tmp_168_type 1414 #define _tmp_169_type 1415 -#define _loop0_170_type 1416 -#define _loop0_171_type 1417 +#define _tmp_170_type 1416 +#define _tmp_171_type 1417 #define _loop0_172_type 1418 -#define _loop1_173_type 1419 -#define _tmp_174_type 1420 -#define _loop0_175_type 1421 +#define _loop0_173_type 1419 +#define _loop0_174_type 1420 +#define _loop1_175_type 1421 #define _tmp_176_type 1422 #define _loop0_177_type 1423 -#define _loop1_178_type 1424 -#define _tmp_179_type 1425 -#define _tmp_180_type 1426 +#define _tmp_178_type 1424 +#define _loop0_179_type 1425 +#define _loop1_180_type 1426 #define _tmp_181_type 1427 -#define _loop0_182_type 1428 +#define _tmp_182_type 1428 #define _tmp_183_type 1429 -#define _tmp_184_type 1430 -#define _loop1_185_type 1431 +#define _loop0_184_type 1430 +#define _tmp_185_type 1431 #define _tmp_186_type 1432 -#define _loop0_187_type 1433 -#define _loop0_188_type 1434 +#define _loop1_187_type 1433 +#define _tmp_188_type 1434 #define _loop0_189_type 1435 -#define _loop0_191_type 1436 -#define _gather_190_type 1437 -#define _tmp_192_type 1438 -#define _loop0_193_type 1439 +#define _loop0_190_type 1436 +#define _loop0_191_type 1437 +#define _loop0_193_type 1438 +#define _gather_192_type 1439 #define _tmp_194_type 1440 #define _loop0_195_type 1441 -#define _loop1_196_type 1442 -#define _loop1_197_type 1443 -#define _tmp_198_type 1444 -#define _tmp_199_type 1445 -#define _loop0_200_type 1446 +#define _tmp_196_type 1442 +#define _loop0_197_type 1443 +#define _loop1_198_type 1444 +#define _loop1_199_type 1445 +#define _tmp_200_type 1446 #define _tmp_201_type 1447 -#define _tmp_202_type 1448 +#define _loop0_202_type 1448 #define _tmp_203_type 1449 -#define _loop0_205_type 1450 -#define _gather_204_type 1451 +#define _tmp_204_type 1450 +#define _tmp_205_type 1451 #define _loop0_207_type 1452 #define _gather_206_type 1453 #define _loop0_209_type 1454 @@ -544,14 +544,14 @@ static char *soft_keywords[] = { #define _gather_210_type 1457 #define _loop0_213_type 1458 #define _gather_212_type 1459 -#define _tmp_214_type 1460 -#define _loop0_215_type 1461 -#define _loop1_216_type 1462 -#define _tmp_217_type 1463 -#define _loop0_218_type 1464 -#define _loop1_219_type 1465 -#define _tmp_220_type 1466 -#define _tmp_221_type 1467 +#define _loop0_215_type 1460 +#define _gather_214_type 1461 +#define _tmp_216_type 1462 +#define _loop0_217_type 1463 +#define _loop1_218_type 1464 +#define _tmp_219_type 1465 +#define _loop0_220_type 1466 +#define _loop1_221_type 1467 #define _tmp_222_type 1468 #define _tmp_223_type 1469 #define _tmp_224_type 1470 @@ -560,10 +560,10 @@ static char *soft_keywords[] = { #define _tmp_227_type 1473 #define _tmp_228_type 1474 #define _tmp_229_type 1475 -#define _loop0_231_type 1476 -#define _gather_230_type 1477 -#define _tmp_232_type 1478 -#define _tmp_233_type 1479 +#define _tmp_230_type 1476 +#define _tmp_231_type 1477 +#define _loop0_233_type 1478 +#define _gather_232_type 1479 #define _tmp_234_type 1480 #define _tmp_235_type 1481 #define _tmp_236_type 1482 @@ -575,9 +575,9 @@ static char *soft_keywords[] = { #define _tmp_242_type 1488 #define _tmp_243_type 1489 #define _tmp_244_type 1490 -#define _loop0_245_type 1491 +#define _tmp_245_type 1491 #define _tmp_246_type 1492 -#define _tmp_247_type 1493 +#define _loop0_247_type 1493 #define _tmp_248_type 1494 #define _tmp_249_type 1495 #define _tmp_250_type 1496 @@ -593,9 +593,9 @@ static char *soft_keywords[] = { #define _tmp_260_type 1506 #define _tmp_261_type 1507 #define _tmp_262_type 1508 -#define _loop0_263_type 1509 +#define _tmp_263_type 1509 #define _tmp_264_type 1510 -#define _tmp_265_type 1511 +#define _loop0_265_type 1511 #define _tmp_266_type 1512 #define _tmp_267_type 1513 #define _tmp_268_type 1514 @@ -609,14 +609,17 @@ static char *soft_keywords[] = { #define _tmp_276_type 1522 #define _tmp_277_type 1523 #define _tmp_278_type 1524 -#define _loop0_280_type 1525 -#define _gather_279_type 1526 +#define _tmp_279_type 1525 +#define _tmp_280_type 1526 #define _tmp_281_type 1527 -#define _tmp_282_type 1528 -#define _tmp_283_type 1529 +#define _loop0_283_type 1528 +#define _gather_282_type 1529 #define _tmp_284_type 1530 #define _tmp_285_type 1531 #define _tmp_286_type 1532 +#define _tmp_287_type 1533 +#define _tmp_288_type 1534 +#define _tmp_289_type 1535 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -1016,8 +1019,8 @@ static asdl_seq *_loop0_149_rule(Parser *p); static asdl_seq *_gather_148_rule(Parser *p); static void *_tmp_150_rule(Parser *p); static void *_tmp_151_rule(Parser *p); -static void *_tmp_152_rule(Parser *p); -static void *_tmp_153_rule(Parser *p); +static asdl_seq *_loop0_153_rule(Parser *p); +static asdl_seq *_gather_152_rule(Parser *p); static void *_tmp_154_rule(Parser *p); static void *_tmp_155_rule(Parser *p); static void *_tmp_156_rule(Parser *p); @@ -1026,50 +1029,50 @@ static void *_tmp_158_rule(Parser *p); static void *_tmp_159_rule(Parser *p); static void *_tmp_160_rule(Parser *p); static void *_tmp_161_rule(Parser *p); -static asdl_seq *_loop0_162_rule(Parser *p); -static asdl_seq *_loop0_163_rule(Parser *p); +static void *_tmp_162_rule(Parser *p); +static void *_tmp_163_rule(Parser *p); static asdl_seq *_loop0_164_rule(Parser *p); -static void *_tmp_165_rule(Parser *p); -static void *_tmp_166_rule(Parser *p); +static asdl_seq *_loop0_165_rule(Parser *p); +static asdl_seq *_loop0_166_rule(Parser *p); static void *_tmp_167_rule(Parser *p); static void *_tmp_168_rule(Parser *p); static void *_tmp_169_rule(Parser *p); -static asdl_seq *_loop0_170_rule(Parser *p); -static asdl_seq *_loop0_171_rule(Parser *p); +static void *_tmp_170_rule(Parser *p); +static void *_tmp_171_rule(Parser *p); static asdl_seq *_loop0_172_rule(Parser *p); -static asdl_seq *_loop1_173_rule(Parser *p); -static void *_tmp_174_rule(Parser *p); -static asdl_seq *_loop0_175_rule(Parser *p); +static asdl_seq *_loop0_173_rule(Parser *p); +static asdl_seq *_loop0_174_rule(Parser *p); +static asdl_seq *_loop1_175_rule(Parser *p); static void *_tmp_176_rule(Parser *p); static asdl_seq *_loop0_177_rule(Parser *p); -static asdl_seq *_loop1_178_rule(Parser *p); -static void *_tmp_179_rule(Parser *p); -static void *_tmp_180_rule(Parser *p); +static void *_tmp_178_rule(Parser *p); +static asdl_seq *_loop0_179_rule(Parser *p); +static asdl_seq *_loop1_180_rule(Parser *p); static void *_tmp_181_rule(Parser *p); -static asdl_seq *_loop0_182_rule(Parser *p); +static void *_tmp_182_rule(Parser *p); static void *_tmp_183_rule(Parser *p); -static void *_tmp_184_rule(Parser *p); -static asdl_seq *_loop1_185_rule(Parser *p); +static asdl_seq *_loop0_184_rule(Parser *p); +static void *_tmp_185_rule(Parser *p); static void *_tmp_186_rule(Parser *p); -static asdl_seq *_loop0_187_rule(Parser *p); -static asdl_seq *_loop0_188_rule(Parser *p); +static asdl_seq *_loop1_187_rule(Parser *p); +static void *_tmp_188_rule(Parser *p); static asdl_seq *_loop0_189_rule(Parser *p); +static asdl_seq *_loop0_190_rule(Parser *p); static asdl_seq *_loop0_191_rule(Parser *p); -static asdl_seq *_gather_190_rule(Parser *p); -static void *_tmp_192_rule(Parser *p); static asdl_seq *_loop0_193_rule(Parser *p); +static asdl_seq *_gather_192_rule(Parser *p); static void *_tmp_194_rule(Parser *p); static asdl_seq *_loop0_195_rule(Parser *p); -static asdl_seq *_loop1_196_rule(Parser *p); -static asdl_seq *_loop1_197_rule(Parser *p); -static void *_tmp_198_rule(Parser *p); -static void *_tmp_199_rule(Parser *p); -static asdl_seq *_loop0_200_rule(Parser *p); +static void *_tmp_196_rule(Parser *p); +static asdl_seq *_loop0_197_rule(Parser *p); +static asdl_seq *_loop1_198_rule(Parser *p); +static asdl_seq *_loop1_199_rule(Parser *p); +static void *_tmp_200_rule(Parser *p); static void *_tmp_201_rule(Parser *p); -static void *_tmp_202_rule(Parser *p); +static asdl_seq *_loop0_202_rule(Parser *p); static void *_tmp_203_rule(Parser *p); -static asdl_seq *_loop0_205_rule(Parser *p); -static asdl_seq *_gather_204_rule(Parser *p); +static void *_tmp_204_rule(Parser *p); +static void *_tmp_205_rule(Parser *p); static asdl_seq *_loop0_207_rule(Parser *p); static asdl_seq *_gather_206_rule(Parser *p); static asdl_seq *_loop0_209_rule(Parser *p); @@ -1078,14 +1081,14 @@ static asdl_seq *_loop0_211_rule(Parser *p); static asdl_seq *_gather_210_rule(Parser *p); static asdl_seq *_loop0_213_rule(Parser *p); static asdl_seq *_gather_212_rule(Parser *p); -static void *_tmp_214_rule(Parser *p); static asdl_seq *_loop0_215_rule(Parser *p); -static asdl_seq *_loop1_216_rule(Parser *p); -static void *_tmp_217_rule(Parser *p); -static asdl_seq *_loop0_218_rule(Parser *p); -static asdl_seq *_loop1_219_rule(Parser *p); -static void *_tmp_220_rule(Parser *p); -static void *_tmp_221_rule(Parser *p); +static asdl_seq *_gather_214_rule(Parser *p); +static void *_tmp_216_rule(Parser *p); +static asdl_seq *_loop0_217_rule(Parser *p); +static asdl_seq *_loop1_218_rule(Parser *p); +static void *_tmp_219_rule(Parser *p); +static asdl_seq *_loop0_220_rule(Parser *p); +static asdl_seq *_loop1_221_rule(Parser *p); static void *_tmp_222_rule(Parser *p); static void *_tmp_223_rule(Parser *p); static void *_tmp_224_rule(Parser *p); @@ -1094,10 +1097,10 @@ static void *_tmp_226_rule(Parser *p); static void *_tmp_227_rule(Parser *p); static void *_tmp_228_rule(Parser *p); static void *_tmp_229_rule(Parser *p); -static asdl_seq *_loop0_231_rule(Parser *p); -static asdl_seq *_gather_230_rule(Parser *p); -static void *_tmp_232_rule(Parser *p); -static void *_tmp_233_rule(Parser *p); +static void *_tmp_230_rule(Parser *p); +static void *_tmp_231_rule(Parser *p); +static asdl_seq *_loop0_233_rule(Parser *p); +static asdl_seq *_gather_232_rule(Parser *p); static void *_tmp_234_rule(Parser *p); static void *_tmp_235_rule(Parser *p); static void *_tmp_236_rule(Parser *p); @@ -1109,9 +1112,9 @@ static void *_tmp_241_rule(Parser *p); static void *_tmp_242_rule(Parser *p); static void *_tmp_243_rule(Parser *p); static void *_tmp_244_rule(Parser *p); -static asdl_seq *_loop0_245_rule(Parser *p); +static void *_tmp_245_rule(Parser *p); static void *_tmp_246_rule(Parser *p); -static void *_tmp_247_rule(Parser *p); +static asdl_seq *_loop0_247_rule(Parser *p); static void *_tmp_248_rule(Parser *p); static void *_tmp_249_rule(Parser *p); static void *_tmp_250_rule(Parser *p); @@ -1127,9 +1130,9 @@ static void *_tmp_259_rule(Parser *p); static void *_tmp_260_rule(Parser *p); static void *_tmp_261_rule(Parser *p); static void *_tmp_262_rule(Parser *p); -static asdl_seq *_loop0_263_rule(Parser *p); +static void *_tmp_263_rule(Parser *p); static void *_tmp_264_rule(Parser *p); -static void *_tmp_265_rule(Parser *p); +static asdl_seq *_loop0_265_rule(Parser *p); static void *_tmp_266_rule(Parser *p); static void *_tmp_267_rule(Parser *p); static void *_tmp_268_rule(Parser *p); @@ -1143,14 +1146,17 @@ static void *_tmp_275_rule(Parser *p); static void *_tmp_276_rule(Parser *p); static void *_tmp_277_rule(Parser *p); static void *_tmp_278_rule(Parser *p); -static asdl_seq *_loop0_280_rule(Parser *p); -static asdl_seq *_gather_279_rule(Parser *p); +static void *_tmp_279_rule(Parser *p); +static void *_tmp_280_rule(Parser *p); static void *_tmp_281_rule(Parser *p); -static void *_tmp_282_rule(Parser *p); -static void *_tmp_283_rule(Parser *p); +static asdl_seq *_loop0_283_rule(Parser *p); +static asdl_seq *_gather_282_rule(Parser *p); static void *_tmp_284_rule(Parser *p); static void *_tmp_285_rule(Parser *p); static void *_tmp_286_rule(Parser *p); +static void *_tmp_287_rule(Parser *p); +static void *_tmp_288_rule(Parser *p); +static void *_tmp_289_rule(Parser *p); // file: statements? $ @@ -17682,7 +17688,7 @@ kwargs_rule(Parser *p) return _res; } -// starred_expression: invalid_starred_expression | '*' expression +// starred_expression: invalid_starred_expression | '*' expression | '*' static expr_ty starred_expression_rule(Parser *p) { @@ -17759,6 +17765,30 @@ starred_expression_rule(Parser *p) D(fprintf(stderr, "%*c%s starred_expression[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' expression")); } + { // '*' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 16)) // token='*' + ) + { + D(fprintf(stderr, "%*c+ starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + _res = RAISE_SYNTAX_ERROR ( "Invalid star expression" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s starred_expression[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); + } _res = NULL; done: p->level--; @@ -19837,7 +19867,7 @@ func_type_comment_rule(Parser *p) } // invalid_arguments: -// | ((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*' +// | ((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+ // | expression for_if_clauses ',' [args | expression for_if_clauses] // | NAME '=' expression for_if_clauses // | [(args ',')] NAME '=' &(',' | ')') @@ -19856,25 +19886,25 @@ invalid_arguments_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*' + { // ((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+ if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*'")); - Token * _literal; + D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); + asdl_seq * _gather_152_var; void *_tmp_151_var; - Token * b; + Token * a; if ( (_tmp_151_var = _tmp_151_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs && - (_literal = _PyPegen_expect_token(p, 12)) // token=',' + (a = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _PyPegen_expect_token(p, 16)) // token='*' + (_gather_152_var = _gather_152_rule(p)) // ','.(starred_expression !'=')+ ) { - D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*'")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( b , "iterable argument unpacking follows keyword argument unpacking" ); + D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "iterable argument unpacking follows keyword argument unpacking" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -19884,7 +19914,7 @@ invalid_arguments_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_arguments[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' '*'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); } { // expression for_if_clauses ',' [args | expression for_if_clauses] if (p->error_indicator) { @@ -19904,7 +19934,7 @@ invalid_arguments_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_152_rule(p), !p->error_indicator) // [args | expression for_if_clauses] + (_opt_var = _tmp_154_rule(p), !p->error_indicator) // [args | expression for_if_clauses] ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); @@ -19964,13 +19994,13 @@ invalid_arguments_rule(Parser *p) expr_ty a; Token * b; if ( - (_opt_var = _tmp_153_rule(p), !p->error_indicator) // [(args ',')] + (_opt_var = _tmp_155_rule(p), !p->error_indicator) // [(args ',')] && (a = _PyPegen_name_token(p)) // NAME && (b = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_154_rule, p) + _PyPegen_lookahead(1, _tmp_156_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(args ',')] NAME '=' &(',' | ')')")); @@ -20108,7 +20138,7 @@ invalid_kwarg_rule(Parser *p) Token* a; Token * b; if ( - (a = (Token*)_tmp_155_rule(p)) // 'True' | 'False' | 'None' + (a = (Token*)_tmp_157_rule(p)) // 'True' | 'False' | 'None' && (b = _PyPegen_expect_token(p, 22)) // token='=' ) @@ -20168,7 +20198,7 @@ invalid_kwarg_rule(Parser *p) expr_ty a; Token * b; if ( - _PyPegen_lookahead(0, _tmp_156_rule, p) + _PyPegen_lookahead(0, _tmp_158_rule, p) && (a = expression_rule(p)) // expression && @@ -20424,7 +20454,7 @@ invalid_expression_rule(Parser *p) expr_ty a; expr_ty b; if ( - _PyPegen_lookahead(0, _tmp_157_rule, p) + _PyPegen_lookahead(0, _tmp_159_rule, p) && (a = disjunction_rule(p)) // disjunction && @@ -20460,7 +20490,7 @@ invalid_expression_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - _PyPegen_lookahead(0, _tmp_158_rule, p) + _PyPegen_lookahead(0, _tmp_160_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); @@ -20581,7 +20611,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_159_rule, p) + _PyPegen_lookahead(0, _tmp_161_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -20607,7 +20637,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, _tmp_160_rule, p) + _PyPegen_lookahead(0, _tmp_162_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -20615,7 +20645,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_161_rule, p) + _PyPegen_lookahead(0, _tmp_163_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -20695,7 +20725,7 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_162_var; + asdl_seq * _loop0_164_var; expr_ty a; expr_ty expression_var; if ( @@ -20703,7 +20733,7 @@ invalid_assignment_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_loop0_162_var = _loop0_162_rule(p)) // star_named_expressions* + (_loop0_164_var = _loop0_164_rule(p)) // star_named_expressions* && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20760,10 +20790,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); Token * _literal; - asdl_seq * _loop0_163_var; + asdl_seq * _loop0_165_var; expr_ty a; if ( - (_loop0_163_var = _loop0_163_rule(p)) // ((star_targets '='))* + (_loop0_165_var = _loop0_165_rule(p)) // ((star_targets '='))* && (a = star_expressions_rule(p)) // star_expressions && @@ -20790,10 +20820,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); Token * _literal; - asdl_seq * _loop0_164_var; + asdl_seq * _loop0_166_var; expr_ty a; if ( - (_loop0_164_var = _loop0_164_rule(p)) // ((star_targets '='))* + (_loop0_166_var = _loop0_166_rule(p)) // ((star_targets '='))* && (a = yield_expr_rule(p)) // yield_expr && @@ -20819,7 +20849,7 @@ invalid_assignment_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); - void *_tmp_165_var; + void *_tmp_167_var; expr_ty a; AugOperator* augassign_var; if ( @@ -20827,7 +20857,7 @@ invalid_assignment_rule(Parser *p) && (augassign_var = augassign_rule(p)) // augassign && - (_tmp_165_var = _tmp_165_rule(p)) // yield_expr | star_expressions + (_tmp_167_var = _tmp_167_rule(p)) // yield_expr | star_expressions ) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions augassign (yield_expr | star_expressions)")); @@ -21049,11 +21079,11 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '(' | '{') starred_expression for_if_clauses")); - void *_tmp_166_var; + void *_tmp_168_var; expr_ty a; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_166_var = _tmp_166_rule(p)) // '[' | '(' | '{' + (_tmp_168_var = _tmp_168_rule(p)) // '[' | '(' | '{' && (a = starred_expression_rule(p)) // starred_expression && @@ -21080,12 +21110,12 @@ invalid_comprehension_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' star_named_expressions for_if_clauses")); Token * _literal; - void *_tmp_167_var; + void *_tmp_169_var; expr_ty a; asdl_expr_seq* b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_167_var = _tmp_167_rule(p)) // '[' | '{' + (_tmp_169_var = _tmp_169_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21115,12 +21145,12 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' for_if_clauses")); - void *_tmp_168_var; + void *_tmp_170_var; expr_ty a; Token * b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_168_var = _tmp_168_rule(p)) // '[' | '{' + (_tmp_170_var = _tmp_170_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21255,13 +21285,13 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); - asdl_seq * _loop0_170_var; - void *_tmp_169_var; + asdl_seq * _loop0_172_var; + void *_tmp_171_var; Token * a; if ( - (_tmp_169_var = _tmp_169_rule(p)) // slash_no_default | slash_with_default + (_tmp_171_var = _tmp_171_rule(p)) // slash_no_default | slash_with_default && - (_loop0_170_var = _loop0_170_rule(p)) // param_maybe_default* + (_loop0_172_var = _loop0_172_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21285,7 +21315,7 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default? param_no_default* invalid_parameters_helper param_no_default")); - asdl_seq * _loop0_171_var; + asdl_seq * _loop0_173_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21293,7 +21323,7 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = slash_no_default_rule(p), !p->error_indicator) // slash_no_default? && - (_loop0_171_var = _loop0_171_rule(p)) // param_no_default* + (_loop0_173_var = _loop0_173_rule(p)) // param_no_default* && (invalid_parameters_helper_var = invalid_parameters_helper_rule(p)) // invalid_parameters_helper && @@ -21319,18 +21349,18 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default* '(' param_no_default+ ','? ')'")); - asdl_seq * _loop0_172_var; - asdl_seq * _loop1_173_var; + asdl_seq * _loop0_174_var; + asdl_seq * _loop1_175_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_172_var = _loop0_172_rule(p)) // param_no_default* + (_loop0_174_var = _loop0_174_rule(p)) // param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_loop1_173_var = _loop1_173_rule(p)) // param_no_default+ + (_loop1_175_var = _loop1_175_rule(p)) // param_no_default+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21357,22 +21387,22 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(slash_no_default | slash_with_default)] param_maybe_default* '*' (',' | param_no_default) param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_175_var; asdl_seq * _loop0_177_var; + asdl_seq * _loop0_179_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_176_var; + void *_tmp_178_var; Token * a; if ( - (_opt_var = _tmp_174_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] + (_opt_var = _tmp_176_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && - (_loop0_175_var = _loop0_175_rule(p)) // param_maybe_default* + (_loop0_177_var = _loop0_177_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_176_var = _tmp_176_rule(p)) // ',' | param_no_default + (_tmp_178_var = _tmp_178_rule(p)) // ',' | param_no_default && - (_loop0_177_var = _loop0_177_rule(p)) // param_maybe_default* + (_loop0_179_var = _loop0_179_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21397,10 +21427,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_178_var; + asdl_seq * _loop1_180_var; Token * a; if ( - (_loop1_178_var = _loop1_178_rule(p)) // param_maybe_default+ + (_loop1_180_var = _loop1_180_rule(p)) // param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -21449,7 +21479,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_179_rule, p) + _PyPegen_lookahead(1, _tmp_181_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -21494,12 +21524,12 @@ invalid_star_etc_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); - void *_tmp_180_var; + void *_tmp_182_var; Token * a; if ( (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_180_var = _tmp_180_rule(p)) // ')' | ',' (')' | '**') + (_tmp_182_var = _tmp_182_rule(p)) // ')' | ',' (')' | '**') ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); @@ -21582,20 +21612,20 @@ invalid_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_182_var; - void *_tmp_181_var; + asdl_seq * _loop0_184_var; void *_tmp_183_var; + void *_tmp_185_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_181_var = _tmp_181_rule(p)) // param_no_default | ',' + (_tmp_183_var = _tmp_183_rule(p)) // param_no_default | ',' && - (_loop0_182_var = _loop0_182_rule(p)) // param_maybe_default* + (_loop0_184_var = _loop0_184_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_183_var = _tmp_183_rule(p)) // param_no_default | ',' + (_tmp_185_var = _tmp_185_rule(p)) // param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); @@ -21710,7 +21740,7 @@ invalid_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_184_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_186_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' ('*' | '**' | '/')")); @@ -21775,13 +21805,13 @@ invalid_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - asdl_seq * _loop1_185_var; + asdl_seq * _loop1_187_var; if ( - (_loop1_185_var = _loop1_185_rule(p)) // param_with_default+ + (_loop1_187_var = _loop1_187_rule(p)) // param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - _res = _loop1_185_var; + _res = _loop1_187_var; goto done; } p->mark = _mark; @@ -21846,13 +21876,13 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); - asdl_seq * _loop0_187_var; - void *_tmp_186_var; + asdl_seq * _loop0_189_var; + void *_tmp_188_var; Token * a; if ( - (_tmp_186_var = _tmp_186_rule(p)) // lambda_slash_no_default | lambda_slash_with_default + (_tmp_188_var = _tmp_188_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && - (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_maybe_default* + (_loop0_189_var = _loop0_189_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21876,7 +21906,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default")); - asdl_seq * _loop0_188_var; + asdl_seq * _loop0_190_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21884,7 +21914,7 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = lambda_slash_no_default_rule(p), !p->error_indicator) // lambda_slash_no_default? && - (_loop0_188_var = _loop0_188_rule(p)) // lambda_param_no_default* + (_loop0_190_var = _loop0_190_rule(p)) // lambda_param_no_default* && (invalid_lambda_parameters_helper_var = invalid_lambda_parameters_helper_rule(p)) // invalid_lambda_parameters_helper && @@ -21910,18 +21940,18 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); - asdl_seq * _gather_190_var; - asdl_seq * _loop0_189_var; + asdl_seq * _gather_192_var; + asdl_seq * _loop0_191_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_189_var = _loop0_189_rule(p)) // lambda_param_no_default* + (_loop0_191_var = _loop0_191_rule(p)) // lambda_param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_190_var = _gather_190_rule(p)) // ','.lambda_param+ + (_gather_192_var = _gather_192_rule(p)) // ','.lambda_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21948,22 +21978,22 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(lambda_slash_no_default | lambda_slash_with_default)] lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_193_var; asdl_seq * _loop0_195_var; + asdl_seq * _loop0_197_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_194_var; + void *_tmp_196_var; Token * a; if ( - (_opt_var = _tmp_192_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] + (_opt_var = _tmp_194_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && - (_loop0_193_var = _loop0_193_rule(p)) // lambda_param_maybe_default* + (_loop0_195_var = _loop0_195_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_194_var = _tmp_194_rule(p)) // ',' | lambda_param_no_default + (_tmp_196_var = _tmp_196_rule(p)) // ',' | lambda_param_no_default && - (_loop0_195_var = _loop0_195_rule(p)) // lambda_param_maybe_default* + (_loop0_197_var = _loop0_197_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21988,10 +22018,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_196_var; + asdl_seq * _loop1_198_var; Token * a; if ( - (_loop1_196_var = _loop1_196_rule(p)) // lambda_param_maybe_default+ + (_loop1_198_var = _loop1_198_rule(p)) // lambda_param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -22062,13 +22092,13 @@ invalid_lambda_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - asdl_seq * _loop1_197_var; + asdl_seq * _loop1_199_var; if ( - (_loop1_197_var = _loop1_197_rule(p)) // lambda_param_with_default+ + (_loop1_199_var = _loop1_199_rule(p)) // lambda_param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - _res = _loop1_197_var; + _res = _loop1_199_var; goto done; } p->mark = _mark; @@ -22104,11 +22134,11 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); Token * _literal; - void *_tmp_198_var; + void *_tmp_200_var; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_198_var = _tmp_198_rule(p)) // ':' | ',' (':' | '**') + (_tmp_200_var = _tmp_200_rule(p)) // ':' | ',' (':' | '**') ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); @@ -22161,20 +22191,20 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_200_var; - void *_tmp_199_var; + asdl_seq * _loop0_202_var; void *_tmp_201_var; + void *_tmp_203_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_199_var = _tmp_199_rule(p)) // lambda_param_no_default | ',' + (_tmp_201_var = _tmp_201_rule(p)) // lambda_param_no_default | ',' && - (_loop0_200_var = _loop0_200_rule(p)) // lambda_param_maybe_default* + (_loop0_202_var = _loop0_202_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_201_var = _tmp_201_rule(p)) // lambda_param_no_default | ',' + (_tmp_203_var = _tmp_203_rule(p)) // lambda_param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); @@ -22292,7 +22322,7 @@ invalid_lambda_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_202_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_204_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' ('*' | '**' | '/')")); @@ -22398,7 +22428,7 @@ invalid_with_item_rule(Parser *p) && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, _tmp_203_rule, p) + _PyPegen_lookahead(1, _tmp_205_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -22571,14 +22601,14 @@ invalid_import_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); - asdl_seq * _gather_204_var; + asdl_seq * _gather_206_var; Token * _keyword; Token * a; expr_ty dotted_name_var; if ( (a = _PyPegen_expect_token(p, 620)) // token='import' && - (_gather_204_var = _gather_204_rule(p)) // ','.dotted_name+ + (_gather_206_var = _gather_206_rule(p)) // ','.dotted_name+ && (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && @@ -22750,7 +22780,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE")); - asdl_seq * _gather_206_var; + asdl_seq * _gather_208_var; Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22760,7 +22790,7 @@ invalid_with_stmt_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 634)) // token='with' && - (_gather_206_var = _gather_206_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_208_var = _gather_208_rule(p)) // ','.(expression ['as' star_target])+ && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -22784,7 +22814,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); - asdl_seq * _gather_208_var; + asdl_seq * _gather_210_var; Token * _keyword; Token * _literal; Token * _literal_1; @@ -22800,7 +22830,7 @@ invalid_with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_208_var = _gather_208_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_210_var = _gather_210_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22849,7 +22879,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); - asdl_seq * _gather_210_var; + asdl_seq * _gather_212_var; Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22860,7 +22890,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (a = _PyPegen_expect_token(p, 634)) // token='with' && - (_gather_210_var = _gather_210_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_212_var = _gather_212_rule(p)) // ','.(expression ['as' star_target])+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22888,7 +22918,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); - asdl_seq * _gather_212_var; + asdl_seq * _gather_214_var; Token * _literal; Token * _literal_1; Token * _literal_2; @@ -22905,7 +22935,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_212_var = _gather_212_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_214_var = _gather_214_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -23002,7 +23032,7 @@ invalid_try_stmt_rule(Parser *p) && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, _tmp_214_rule, p) + _PyPegen_lookahead(0, _tmp_216_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -23027,8 +23057,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_215_var; - asdl_seq * _loop1_216_var; + asdl_seq * _loop0_217_var; + asdl_seq * _loop1_218_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -23039,9 +23069,9 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_215_var = _loop0_215_rule(p)) // block* + (_loop0_217_var = _loop0_217_rule(p)) // block* && - (_loop1_216_var = _loop1_216_rule(p)) // except_block+ + (_loop1_218_var = _loop1_218_rule(p)) // except_block+ && (a = _PyPegen_expect_token(p, 656)) // token='except' && @@ -23049,7 +23079,7 @@ invalid_try_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_217_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_219_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23076,8 +23106,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_218_var; - asdl_seq * _loop1_219_var; + asdl_seq * _loop0_220_var; + asdl_seq * _loop1_221_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -23086,13 +23116,13 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_218_var = _loop0_218_rule(p)) // block* + (_loop0_220_var = _loop0_220_rule(p)) // block* && - (_loop1_219_var = _loop1_219_rule(p)) // except_star_block+ + (_loop1_221_var = _loop1_221_rule(p)) // except_star_block+ && (a = _PyPegen_expect_token(p, 656)) // token='except' && - (_opt_var = _tmp_220_rule(p), !p->error_indicator) // [expression ['as' NAME]] + (_opt_var = _tmp_222_rule(p), !p->error_indicator) // [expression ['as' NAME]] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23159,7 +23189,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var_1 = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_223_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23197,7 +23227,7 @@ invalid_except_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var_1 = _tmp_222_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23249,14 +23279,14 @@ invalid_except_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_223_var; + void *_tmp_225_var; Token * a; if ( (a = _PyPegen_expect_token(p, 656)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_223_var = _tmp_223_rule(p)) // NEWLINE | ':' + (_tmp_225_var = _tmp_225_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); @@ -23361,7 +23391,7 @@ invalid_except_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_226_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23455,7 +23485,7 @@ invalid_except_star_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_225_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_227_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23819,7 +23849,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_226_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_228_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -24311,7 +24341,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_3 = _tmp_227_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_3 = _tmp_229_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24374,7 +24404,7 @@ invalid_class_def_raw_rule(Parser *p) && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_228_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_230_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24413,7 +24443,7 @@ invalid_class_def_raw_rule(Parser *p) && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_229_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_231_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24463,11 +24493,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_230_var; + asdl_seq * _gather_232_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_230_var = _gather_230_rule(p)) // ','.double_starred_kvpair+ + (_gather_232_var = _gather_232_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24475,7 +24505,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_230_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_232_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -24528,7 +24558,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_232_rule, p) + _PyPegen_lookahead(1, _tmp_234_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24638,7 +24668,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_233_rule, p) + _PyPegen_lookahead(1, _tmp_235_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24854,7 +24884,7 @@ invalid_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, _tmp_234_rule, p) + _PyPegen_lookahead(0, _tmp_236_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !(yield_expr | star_expressions)")); @@ -24877,13 +24907,13 @@ invalid_replacement_field_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); Token * _literal; - void *_tmp_235_var; + void *_tmp_237_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_235_var = _tmp_235_rule(p)) // yield_expr | star_expressions + (_tmp_237_var = _tmp_237_rule(p)) // yield_expr | star_expressions && - _PyPegen_lookahead(0, _tmp_236_rule, p) + _PyPegen_lookahead(0, _tmp_238_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}')")); @@ -24907,15 +24937,15 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); Token * _literal; Token * _literal_1; - void *_tmp_237_var; + void *_tmp_239_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_237_var = _tmp_237_rule(p)) // yield_expr | star_expressions + (_tmp_239_var = _tmp_239_rule(p)) // yield_expr | star_expressions && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, _tmp_238_rule, p) + _PyPegen_lookahead(0, _tmp_240_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '=' !('!' | ':' | '}')")); @@ -24940,12 +24970,12 @@ invalid_replacement_field_rule(Parser *p) Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_239_var; + void *_tmp_241_var; void *invalid_conversion_character_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_239_var = _tmp_239_rule(p)) // yield_expr | star_expressions + (_tmp_241_var = _tmp_241_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && @@ -24953,7 +24983,7 @@ invalid_replacement_field_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? invalid_conversion_character")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_239_var, _opt_var, invalid_conversion_character_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_241_var, _opt_var, invalid_conversion_character_var); goto done; } p->mark = _mark; @@ -24971,17 +25001,17 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_240_var; + void *_tmp_242_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_240_var = _tmp_240_rule(p)) // yield_expr | star_expressions + (_tmp_242_var = _tmp_242_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_241_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_243_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, _tmp_242_rule, p) + _PyPegen_lookahead(0, _tmp_244_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}')")); @@ -25005,24 +25035,24 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_245_var; + asdl_seq * _loop0_247_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_243_var; + void *_tmp_245_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_243_var = _tmp_243_rule(p)) // yield_expr | star_expressions + (_tmp_245_var = _tmp_245_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_244_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_246_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_245_var = _loop0_245_rule(p)) // fstring_format_spec* + (_loop0_247_var = _loop0_247_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25051,15 +25081,15 @@ invalid_replacement_field_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings - void *_tmp_246_var; + void *_tmp_248_var; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (_tmp_246_var = _tmp_246_rule(p)) // yield_expr | star_expressions + (_tmp_248_var = _tmp_248_rule(p)) // yield_expr | star_expressions && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_247_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_249_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25106,7 +25136,7 @@ invalid_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, _tmp_248_rule, p) + _PyPegen_lookahead(1, _tmp_250_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -25173,14 +25203,14 @@ invalid_arithmetic_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arithmetic[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); - void *_tmp_249_var; + void *_tmp_251_var; Token * a; expr_ty b; expr_ty sum_var; if ( (sum_var = sum_rule(p)) // sum && - (_tmp_249_var = _tmp_249_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' + (_tmp_251_var = _tmp_251_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && (a = _PyPegen_expect_token(p, 678)) // token='not' && @@ -25225,11 +25255,11 @@ invalid_factor_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_factor[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); - void *_tmp_250_var; + void *_tmp_252_var; Token * a; expr_ty b; if ( - (_tmp_250_var = _tmp_250_rule(p)) // '+' | '-' | '~' + (_tmp_252_var = _tmp_252_rule(p)) // '+' | '-' | '~' && (a = _PyPegen_expect_token(p, 678)) // token='not' && @@ -26070,12 +26100,12 @@ _loop1_14_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_251_var; + void *_tmp_253_var; while ( - (_tmp_251_var = _tmp_251_rule(p)) // star_targets '=' + (_tmp_253_var = _tmp_253_rule(p)) // star_targets '=' ) { - _res = _tmp_251_var; + _res = _tmp_253_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26639,12 +26669,12 @@ _loop0_24_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_252_var; + void *_tmp_254_var; while ( - (_tmp_252_var = _tmp_252_rule(p)) // '.' | '...' + (_tmp_254_var = _tmp_254_rule(p)) // '.' | '...' ) { - _res = _tmp_252_var; + _res = _tmp_254_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26706,12 +26736,12 @@ _loop1_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_253_var; + void *_tmp_255_var; while ( - (_tmp_253_var = _tmp_253_rule(p)) // '.' | '...' + (_tmp_255_var = _tmp_255_rule(p)) // '.' | '...' ) { - _res = _tmp_253_var; + _res = _tmp_255_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27104,12 +27134,12 @@ _loop1_32_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_254_var; + void *_tmp_256_var; while ( - (_tmp_254_var = _tmp_254_rule(p)) // '@' named_expression NEWLINE + (_tmp_256_var = _tmp_256_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_254_var; + _res = _tmp_256_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30234,12 +30264,12 @@ _loop1_82_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_255_var; + void *_tmp_257_var; while ( - (_tmp_255_var = _tmp_255_rule(p)) // ',' expression + (_tmp_257_var = _tmp_257_rule(p)) // ',' expression ) { - _res = _tmp_255_var; + _res = _tmp_257_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30306,12 +30336,12 @@ _loop1_83_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_256_var; + void *_tmp_258_var; while ( - (_tmp_256_var = _tmp_256_rule(p)) // ',' star_expression + (_tmp_258_var = _tmp_258_rule(p)) // ',' star_expression ) { - _res = _tmp_256_var; + _res = _tmp_258_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30495,12 +30525,12 @@ _loop1_86_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_257_var; + void *_tmp_259_var; while ( - (_tmp_257_var = _tmp_257_rule(p)) // 'or' conjunction + (_tmp_259_var = _tmp_259_rule(p)) // 'or' conjunction ) { - _res = _tmp_257_var; + _res = _tmp_259_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30567,12 +30597,12 @@ _loop1_87_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_258_var; + void *_tmp_260_var; while ( - (_tmp_258_var = _tmp_258_rule(p)) // 'and' inversion + (_tmp_260_var = _tmp_260_rule(p)) // 'and' inversion ) { - _res = _tmp_258_var; + _res = _tmp_260_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30759,7 +30789,7 @@ _loop0_91_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_259_rule(p)) // slice | starred_expression + (elem = _tmp_261_rule(p)) // slice | starred_expression ) { _res = elem; @@ -30824,7 +30854,7 @@ _gather_90_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_259_rule(p)) // slice | starred_expression + (elem = _tmp_261_rule(p)) // slice | starred_expression && (seq = _loop0_91_rule(p)) // _loop0_91 ) @@ -32423,12 +32453,12 @@ _loop1_115_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); - void *_tmp_260_var; + void *_tmp_262_var; while ( - (_tmp_260_var = _tmp_260_rule(p)) // fstring | string + (_tmp_262_var = _tmp_262_rule(p)) // fstring | string ) { - _res = _tmp_260_var; + _res = _tmp_262_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32733,12 +32763,12 @@ _loop0_120_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_261_var; + void *_tmp_263_var; while ( - (_tmp_261_var = _tmp_261_rule(p)) // 'if' disjunction + (_tmp_263_var = _tmp_263_rule(p)) // 'if' disjunction ) { - _res = _tmp_261_var; + _res = _tmp_263_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32800,12 +32830,12 @@ _loop0_121_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_262_var; + void *_tmp_264_var; while ( - (_tmp_262_var = _tmp_262_rule(p)) // 'if' disjunction + (_tmp_264_var = _tmp_264_rule(p)) // 'if' disjunction ) { - _res = _tmp_262_var; + _res = _tmp_264_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32858,20 +32888,20 @@ _tmp_122_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - asdl_seq * _loop0_263_var; + asdl_seq * _loop0_265_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty bitwise_or_var; if ( (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - (_loop0_263_var = _loop0_263_rule(p)) // ((',' bitwise_or))* + (_loop0_265_var = _loop0_265_rule(p)) // ((',' bitwise_or))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_263_var, _opt_var); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_265_var, _opt_var); goto done; } p->mark = _mark; @@ -32976,7 +33006,7 @@ _loop0_125_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_264_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_266_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -33042,7 +33072,7 @@ _gather_124_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_264_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_266_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && (seq = _loop0_125_rule(p)) // _loop0_125 ) @@ -33603,12 +33633,12 @@ _loop0_135_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_265_var; + void *_tmp_267_var; while ( - (_tmp_265_var = _tmp_265_rule(p)) // ',' star_target + (_tmp_267_var = _tmp_267_rule(p)) // ',' star_target ) { - _res = _tmp_265_var; + _res = _tmp_267_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33787,12 +33817,12 @@ _loop1_138_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_266_var; + void *_tmp_268_var; while ( - (_tmp_266_var = _tmp_266_rule(p)) // ',' star_target + (_tmp_268_var = _tmp_268_rule(p)) // ',' star_target ) { - _res = _tmp_266_var; + _res = _tmp_268_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34518,13 +34548,13 @@ _tmp_151_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_267_var; + void *_tmp_269_var; if ( - (_tmp_267_var = _tmp_267_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + (_tmp_269_var = _tmp_269_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_267_var; + _res = _tmp_269_var; goto done; } p->mark = _mark; @@ -34556,9 +34586,126 @@ _tmp_151_rule(Parser *p) return _res; } -// _tmp_152: args | expression for_if_clauses +// _loop0_153: ',' (starred_expression !'=') +static asdl_seq * +_loop0_153_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' (starred_expression !'=') + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); + Token * _literal; + void *elem; + while ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = _tmp_270_rule(p)) // starred_expression !'=' + ) + { + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_153[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression !'=')")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _gather_152: (starred_expression !'=') _loop0_153 +static asdl_seq * +_gather_152_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + asdl_seq * _res = NULL; + int _mark = p->mark; + { // (starred_expression !'=') _loop0_153 + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _gather_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_153")); + void *elem; + asdl_seq * seq; + if ( + (elem = _tmp_270_rule(p)) // starred_expression !'=' + && + (seq = _loop0_153_rule(p)) // _loop0_153 + ) + { + D(fprintf(stderr, "%*c+ _gather_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_153")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _gather_152[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_153")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_154: args | expression for_if_clauses static void * -_tmp_152_rule(Parser *p) +_tmp_154_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34574,18 +34721,18 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); expr_ty args_var; if ( (args_var = args_rule(p)) // args ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); _res = args_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args")); } { // expression for_if_clauses @@ -34593,7 +34740,7 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); expr_ty expression_var; asdl_comprehension_seq* for_if_clauses_var; if ( @@ -34602,12 +34749,12 @@ _tmp_152_rule(Parser *p) (for_if_clauses_var = for_if_clauses_rule(p)) // for_if_clauses ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); _res = _PyPegen_dummy_name(p, expression_var, for_if_clauses_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression for_if_clauses")); } _res = NULL; @@ -34616,9 +34763,9 @@ _tmp_152_rule(Parser *p) return _res; } -// _tmp_153: args ',' +// _tmp_155: args ',' static void * -_tmp_153_rule(Parser *p) +_tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34634,7 +34781,7 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); Token * _literal; expr_ty args_var; if ( @@ -34643,12 +34790,12 @@ _tmp_153_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); _res = _PyPegen_dummy_name(p, args_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args ','")); } _res = NULL; @@ -34657,9 +34804,9 @@ _tmp_153_rule(Parser *p) return _res; } -// _tmp_154: ',' | ')' +// _tmp_156: ',' | ')' static void * -_tmp_154_rule(Parser *p) +_tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34675,18 +34822,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -34694,18 +34841,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } _res = NULL; @@ -34714,9 +34861,9 @@ _tmp_154_rule(Parser *p) return _res; } -// _tmp_155: 'True' | 'False' | 'None' +// _tmp_157: 'True' | 'False' | 'None' static void * -_tmp_155_rule(Parser *p) +_tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34732,18 +34879,18 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 613)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'False' @@ -34751,18 +34898,18 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 615)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } { // 'None' @@ -34770,18 +34917,18 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 614)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } _res = NULL; @@ -34790,9 +34937,9 @@ _tmp_155_rule(Parser *p) return _res; } -// _tmp_156: NAME '=' +// _tmp_158: NAME '=' static void * -_tmp_156_rule(Parser *p) +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34808,7 +34955,7 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); Token * _literal; expr_ty name_var; if ( @@ -34817,12 +34964,12 @@ _tmp_156_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); _res = _PyPegen_dummy_name(p, name_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME '='")); } _res = NULL; @@ -34831,9 +34978,9 @@ _tmp_156_rule(Parser *p) return _res; } -// _tmp_157: NAME STRING | SOFT_KEYWORD +// _tmp_159: NAME STRING | SOFT_KEYWORD static void * -_tmp_157_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34849,7 +34996,7 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); expr_ty name_var; expr_ty string_var; if ( @@ -34858,12 +35005,12 @@ _tmp_157_rule(Parser *p) (string_var = _PyPegen_string_token(p)) // STRING ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); _res = _PyPegen_dummy_name(p, name_var, string_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME STRING")); } { // SOFT_KEYWORD @@ -34871,18 +35018,18 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); expr_ty soft_keyword_var; if ( (soft_keyword_var = _PyPegen_soft_keyword_token(p)) // SOFT_KEYWORD ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); _res = soft_keyword_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "SOFT_KEYWORD")); } _res = NULL; @@ -34891,9 +35038,9 @@ _tmp_157_rule(Parser *p) return _res; } -// _tmp_158: 'else' | ':' +// _tmp_160: 'else' | ':' static void * -_tmp_158_rule(Parser *p) +_tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34909,18 +35056,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 664)) // token='else' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'else'")); } { // ':' @@ -34928,18 +35075,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -34948,9 +35095,9 @@ _tmp_158_rule(Parser *p) return _res; } -// _tmp_159: '=' | ':=' +// _tmp_161: '=' | ':=' static void * -_tmp_159_rule(Parser *p) +_tmp_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34966,18 +35113,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34985,18 +35132,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -35005,9 +35152,9 @@ _tmp_159_rule(Parser *p) return _res; } -// _tmp_160: list | tuple | genexp | 'True' | 'None' | 'False' +// _tmp_162: list | tuple | genexp | 'True' | 'None' | 'False' static void * -_tmp_160_rule(Parser *p) +_tmp_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35023,18 +35170,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // tuple @@ -35042,18 +35189,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // genexp @@ -35061,18 +35208,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } { // 'True' @@ -35080,18 +35227,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 613)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'None' @@ -35099,18 +35246,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 614)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } { // 'False' @@ -35118,18 +35265,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 615)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; @@ -35138,9 +35285,9 @@ _tmp_160_rule(Parser *p) return _res; } -// _tmp_161: '=' | ':=' +// _tmp_163: '=' | ':=' static void * -_tmp_161_rule(Parser *p) +_tmp_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35156,18 +35303,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -35175,18 +35322,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -35195,9 +35342,9 @@ _tmp_161_rule(Parser *p) return _res; } -// _loop0_162: star_named_expressions +// _loop0_164: star_named_expressions static asdl_seq * -_loop0_162_rule(Parser *p) +_loop0_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35222,7 +35369,7 @@ _loop0_162_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); + D(fprintf(stderr, "%*c> _loop0_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); asdl_expr_seq* star_named_expressions_var; while ( (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions @@ -35245,7 +35392,7 @@ _loop0_162_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35262,9 +35409,9 @@ _loop0_162_rule(Parser *p) return _seq; } -// _loop0_163: (star_targets '=') +// _loop0_165: (star_targets '=') static asdl_seq * -_loop0_163_rule(Parser *p) +_loop0_165_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35289,13 +35436,13 @@ _loop0_163_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_268_var; + D(fprintf(stderr, "%*c> _loop0_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_271_var; while ( - (_tmp_268_var = _tmp_268_rule(p)) // star_targets '=' + (_tmp_271_var = _tmp_271_rule(p)) // star_targets '=' ) { - _res = _tmp_268_var; + _res = _tmp_271_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35312,7 +35459,7 @@ _loop0_163_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35329,9 +35476,9 @@ _loop0_163_rule(Parser *p) return _seq; } -// _loop0_164: (star_targets '=') +// _loop0_166: (star_targets '=') static asdl_seq * -_loop0_164_rule(Parser *p) +_loop0_166_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35356,13 +35503,13 @@ _loop0_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_269_var; + D(fprintf(stderr, "%*c> _loop0_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_272_var; while ( - (_tmp_269_var = _tmp_269_rule(p)) // star_targets '=' + (_tmp_272_var = _tmp_272_rule(p)) // star_targets '=' ) { - _res = _tmp_269_var; + _res = _tmp_272_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35379,7 +35526,7 @@ _loop0_164_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35396,9 +35543,9 @@ _loop0_164_rule(Parser *p) return _seq; } -// _tmp_165: yield_expr | star_expressions +// _tmp_167: yield_expr | star_expressions static void * -_tmp_165_rule(Parser *p) +_tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35414,18 +35561,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -35433,18 +35580,18 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -35453,9 +35600,9 @@ _tmp_165_rule(Parser *p) return _res; } -// _tmp_166: '[' | '(' | '{' +// _tmp_168: '[' | '(' | '{' static void * -_tmp_166_rule(Parser *p) +_tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35471,18 +35618,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '(' @@ -35490,18 +35637,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '{' @@ -35509,18 +35656,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35529,9 +35676,9 @@ _tmp_166_rule(Parser *p) return _res; } -// _tmp_167: '[' | '{' +// _tmp_169: '[' | '{' static void * -_tmp_167_rule(Parser *p) +_tmp_169_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35547,18 +35694,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35566,18 +35713,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35586,9 +35733,9 @@ _tmp_167_rule(Parser *p) return _res; } -// _tmp_168: '[' | '{' +// _tmp_170: '[' | '{' static void * -_tmp_168_rule(Parser *p) +_tmp_170_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35604,18 +35751,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35623,18 +35770,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35643,9 +35790,9 @@ _tmp_168_rule(Parser *p) return _res; } -// _tmp_169: slash_no_default | slash_with_default +// _tmp_171: slash_no_default | slash_with_default static void * -_tmp_169_rule(Parser *p) +_tmp_171_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35661,18 +35808,18 @@ _tmp_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_171[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -35680,18 +35827,18 @@ _tmp_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_171[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -35700,9 +35847,9 @@ _tmp_169_rule(Parser *p) return _res; } -// _loop0_170: param_maybe_default +// _loop0_172: param_maybe_default static asdl_seq * -_loop0_170_rule(Parser *p) +_loop0_172_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35727,7 +35874,7 @@ _loop0_170_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -35750,7 +35897,7 @@ _loop0_170_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35767,9 +35914,9 @@ _loop0_170_rule(Parser *p) return _seq; } -// _loop0_171: param_no_default +// _loop0_173: param_no_default static asdl_seq * -_loop0_171_rule(Parser *p) +_loop0_173_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35794,7 +35941,7 @@ _loop0_171_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35817,7 +35964,7 @@ _loop0_171_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35834,9 +35981,9 @@ _loop0_171_rule(Parser *p) return _seq; } -// _loop0_172: param_no_default +// _loop0_174: param_no_default static asdl_seq * -_loop0_172_rule(Parser *p) +_loop0_174_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35861,7 +36008,7 @@ _loop0_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35884,7 +36031,7 @@ _loop0_172_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35901,9 +36048,9 @@ _loop0_172_rule(Parser *p) return _seq; } -// _loop1_173: param_no_default +// _loop1_175: param_no_default static asdl_seq * -_loop1_173_rule(Parser *p) +_loop1_175_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35928,7 +36075,7 @@ _loop1_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop1_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -35951,7 +36098,7 @@ _loop1_173_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_175[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } if (_n == 0 || p->error_indicator) { @@ -35973,9 +36120,9 @@ _loop1_173_rule(Parser *p) return _seq; } -// _tmp_174: slash_no_default | slash_with_default +// _tmp_176: slash_no_default | slash_with_default static void * -_tmp_174_rule(Parser *p) +_tmp_176_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35991,18 +36138,18 @@ _tmp_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -36010,18 +36157,18 @@ _tmp_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -36030,9 +36177,9 @@ _tmp_174_rule(Parser *p) return _res; } -// _loop0_175: param_maybe_default +// _loop0_177: param_maybe_default static asdl_seq * -_loop0_175_rule(Parser *p) +_loop0_177_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36057,7 +36204,7 @@ _loop0_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36080,7 +36227,7 @@ _loop0_175_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36097,9 +36244,9 @@ _loop0_175_rule(Parser *p) return _seq; } -// _tmp_176: ',' | param_no_default +// _tmp_178: ',' | param_no_default static void * -_tmp_176_rule(Parser *p) +_tmp_178_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36115,18 +36262,18 @@ _tmp_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // param_no_default @@ -36134,18 +36281,18 @@ _tmp_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } _res = NULL; @@ -36154,9 +36301,9 @@ _tmp_176_rule(Parser *p) return _res; } -// _loop0_177: param_maybe_default +// _loop0_179: param_maybe_default static asdl_seq * -_loop0_177_rule(Parser *p) +_loop0_179_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36181,7 +36328,7 @@ _loop0_177_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36204,7 +36351,7 @@ _loop0_177_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_177[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_179[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36221,9 +36368,9 @@ _loop0_177_rule(Parser *p) return _seq; } -// _loop1_178: param_maybe_default +// _loop1_180: param_maybe_default static asdl_seq * -_loop1_178_rule(Parser *p) +_loop1_180_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36248,7 +36395,7 @@ _loop1_178_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36271,7 +36418,7 @@ _loop1_178_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_178[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_180[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -36293,9 +36440,9 @@ _loop1_178_rule(Parser *p) return _seq; } -// _tmp_179: ')' | ',' +// _tmp_181: ')' | ',' static void * -_tmp_179_rule(Parser *p) +_tmp_181_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36311,18 +36458,18 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' @@ -36330,18 +36477,18 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36350,9 +36497,9 @@ _tmp_179_rule(Parser *p) return _res; } -// _tmp_180: ')' | ',' (')' | '**') +// _tmp_182: ')' | ',' (')' | '**') static void * -_tmp_180_rule(Parser *p) +_tmp_182_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36368,18 +36515,18 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' (')' | '**') @@ -36387,21 +36534,21 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_270_var; + void *_tmp_273_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_270_var = _tmp_270_rule(p)) // ')' | '**' + (_tmp_273_var = _tmp_273_rule(p)) // ')' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_270_var); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_273_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (')' | '**')")); } _res = NULL; @@ -36410,9 +36557,9 @@ _tmp_180_rule(Parser *p) return _res; } -// _tmp_181: param_no_default | ',' +// _tmp_183: param_no_default | ',' static void * -_tmp_181_rule(Parser *p) +_tmp_183_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36428,18 +36575,18 @@ _tmp_181_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36447,18 +36594,18 @@ _tmp_181_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36467,9 +36614,9 @@ _tmp_181_rule(Parser *p) return _res; } -// _loop0_182: param_maybe_default +// _loop0_184: param_maybe_default static asdl_seq * -_loop0_182_rule(Parser *p) +_loop0_184_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36494,7 +36641,7 @@ _loop0_182_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36517,7 +36664,7 @@ _loop0_182_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_182[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_184[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36534,9 +36681,9 @@ _loop0_182_rule(Parser *p) return _seq; } -// _tmp_183: param_no_default | ',' +// _tmp_185: param_no_default | ',' static void * -_tmp_183_rule(Parser *p) +_tmp_185_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36552,18 +36699,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36571,18 +36718,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36591,9 +36738,9 @@ _tmp_183_rule(Parser *p) return _res; } -// _tmp_184: '*' | '**' | '/' +// _tmp_186: '*' | '**' | '/' static void * -_tmp_184_rule(Parser *p) +_tmp_186_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36609,18 +36756,18 @@ _tmp_184_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_186[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_186[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -36628,18 +36775,18 @@ _tmp_184_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_186[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_186[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -36647,18 +36794,18 @@ _tmp_184_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_186[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_186[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -36667,9 +36814,9 @@ _tmp_184_rule(Parser *p) return _res; } -// _loop1_185: param_with_default +// _loop1_187: param_with_default static asdl_seq * -_loop1_185_rule(Parser *p) +_loop1_187_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36694,7 +36841,7 @@ _loop1_185_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop1_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -36717,7 +36864,7 @@ _loop1_185_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_185[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_187[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -36739,9 +36886,9 @@ _loop1_185_rule(Parser *p) return _seq; } -// _tmp_186: lambda_slash_no_default | lambda_slash_with_default +// _tmp_188: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_186_rule(Parser *p) +_tmp_188_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36757,18 +36904,18 @@ _tmp_186_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_186[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_188[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_186[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_188[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -36776,18 +36923,18 @@ _tmp_186_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_186[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_188[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_186[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_188[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -36796,9 +36943,9 @@ _tmp_186_rule(Parser *p) return _res; } -// _loop0_187: lambda_param_maybe_default +// _loop0_189: lambda_param_maybe_default static asdl_seq * -_loop0_187_rule(Parser *p) +_loop0_189_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36823,7 +36970,7 @@ _loop0_187_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -36846,7 +36993,7 @@ _loop0_187_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_189[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36863,9 +37010,9 @@ _loop0_187_rule(Parser *p) return _seq; } -// _loop0_188: lambda_param_no_default +// _loop0_190: lambda_param_no_default static asdl_seq * -_loop0_188_rule(Parser *p) +_loop0_190_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36890,7 +37037,7 @@ _loop0_188_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -36913,7 +37060,7 @@ _loop0_188_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_188[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_190[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36930,9 +37077,9 @@ _loop0_188_rule(Parser *p) return _seq; } -// _loop0_189: lambda_param_no_default +// _loop0_191: lambda_param_no_default static asdl_seq * -_loop0_189_rule(Parser *p) +_loop0_191_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36957,7 +37104,7 @@ _loop0_189_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -36980,7 +37127,7 @@ _loop0_189_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_189[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_191[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36997,9 +37144,9 @@ _loop0_189_rule(Parser *p) return _seq; } -// _loop0_191: ',' lambda_param +// _loop0_193: ',' lambda_param static asdl_seq * -_loop0_191_rule(Parser *p) +_loop0_193_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37024,7 +37171,7 @@ _loop0_191_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); + D(fprintf(stderr, "%*c> _loop0_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); Token * _literal; arg_ty elem; while ( @@ -37056,7 +37203,7 @@ _loop0_191_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_191[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_193[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' lambda_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37073,9 +37220,9 @@ _loop0_191_rule(Parser *p) return _seq; } -// _gather_190: lambda_param _loop0_191 +// _gather_192: lambda_param _loop0_193 static asdl_seq * -_gather_190_rule(Parser *p) +_gather_192_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37086,27 +37233,27 @@ _gather_190_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // lambda_param _loop0_191 + { // lambda_param _loop0_193 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_191")); + D(fprintf(stderr, "%*c> _gather_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_193")); arg_ty elem; asdl_seq * seq; if ( (elem = lambda_param_rule(p)) // lambda_param && - (seq = _loop0_191_rule(p)) // _loop0_191 + (seq = _loop0_193_rule(p)) // _loop0_193 ) { - D(fprintf(stderr, "%*c+ _gather_190[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_191")); + D(fprintf(stderr, "%*c+ _gather_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_193")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_190[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_191")); + D(fprintf(stderr, "%*c%s _gather_192[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_193")); } _res = NULL; done: @@ -37114,9 +37261,9 @@ _gather_190_rule(Parser *p) return _res; } -// _tmp_192: lambda_slash_no_default | lambda_slash_with_default +// _tmp_194: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_192_rule(Parser *p) +_tmp_194_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37132,18 +37279,18 @@ _tmp_192_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_194[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_194[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -37151,18 +37298,18 @@ _tmp_192_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_194[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_194[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -37171,9 +37318,9 @@ _tmp_192_rule(Parser *p) return _res; } -// _loop0_193: lambda_param_maybe_default +// _loop0_195: lambda_param_maybe_default static asdl_seq * -_loop0_193_rule(Parser *p) +_loop0_195_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37198,7 +37345,7 @@ _loop0_193_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37221,7 +37368,7 @@ _loop0_193_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_193[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_195[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37238,9 +37385,9 @@ _loop0_193_rule(Parser *p) return _seq; } -// _tmp_194: ',' | lambda_param_no_default +// _tmp_196: ',' | lambda_param_no_default static void * -_tmp_194_rule(Parser *p) +_tmp_196_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37256,18 +37403,18 @@ _tmp_194_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_194[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_196[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_194[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_196[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // lambda_param_no_default @@ -37275,18 +37422,18 @@ _tmp_194_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_194[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_196[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_194[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_196[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } _res = NULL; @@ -37295,9 +37442,9 @@ _tmp_194_rule(Parser *p) return _res; } -// _loop0_195: lambda_param_maybe_default +// _loop0_197: lambda_param_maybe_default static asdl_seq * -_loop0_195_rule(Parser *p) +_loop0_197_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37322,7 +37469,7 @@ _loop0_195_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37345,7 +37492,7 @@ _loop0_195_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_195[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_197[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37362,9 +37509,9 @@ _loop0_195_rule(Parser *p) return _seq; } -// _loop1_196: lambda_param_maybe_default +// _loop1_198: lambda_param_maybe_default static asdl_seq * -_loop1_196_rule(Parser *p) +_loop1_198_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37389,7 +37536,7 @@ _loop1_196_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37412,7 +37559,7 @@ _loop1_196_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_196[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_198[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -37434,9 +37581,9 @@ _loop1_196_rule(Parser *p) return _seq; } -// _loop1_197: lambda_param_with_default +// _loop1_199: lambda_param_with_default static asdl_seq * -_loop1_197_rule(Parser *p) +_loop1_199_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37461,7 +37608,7 @@ _loop1_197_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c> _loop1_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); NameDefaultPair* lambda_param_with_default_var; while ( (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default @@ -37484,7 +37631,7 @@ _loop1_197_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_197[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_199[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -37506,9 +37653,9 @@ _loop1_197_rule(Parser *p) return _seq; } -// _tmp_198: ':' | ',' (':' | '**') +// _tmp_200: ':' | ',' (':' | '**') static void * -_tmp_198_rule(Parser *p) +_tmp_200_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37524,18 +37671,18 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // ',' (':' | '**') @@ -37543,21 +37690,21 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_271_var; + void *_tmp_274_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_271_var = _tmp_271_rule(p)) // ':' | '**' + (_tmp_274_var = _tmp_274_rule(p)) // ':' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_271_var); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_274_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (':' | '**')")); } _res = NULL; @@ -37566,9 +37713,9 @@ _tmp_198_rule(Parser *p) return _res; } -// _tmp_199: lambda_param_no_default | ',' +// _tmp_201: lambda_param_no_default | ',' static void * -_tmp_199_rule(Parser *p) +_tmp_201_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37584,18 +37731,18 @@ _tmp_199_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -37603,18 +37750,18 @@ _tmp_199_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37623,9 +37770,9 @@ _tmp_199_rule(Parser *p) return _res; } -// _loop0_200: lambda_param_maybe_default +// _loop0_202: lambda_param_maybe_default static asdl_seq * -_loop0_200_rule(Parser *p) +_loop0_202_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37650,7 +37797,7 @@ _loop0_200_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37673,7 +37820,7 @@ _loop0_200_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_200[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_202[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37690,9 +37837,9 @@ _loop0_200_rule(Parser *p) return _seq; } -// _tmp_201: lambda_param_no_default | ',' +// _tmp_203: lambda_param_no_default | ',' static void * -_tmp_201_rule(Parser *p) +_tmp_203_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37708,18 +37855,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -37727,18 +37874,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37747,9 +37894,9 @@ _tmp_201_rule(Parser *p) return _res; } -// _tmp_202: '*' | '**' | '/' +// _tmp_204: '*' | '**' | '/' static void * -_tmp_202_rule(Parser *p) +_tmp_204_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37765,18 +37912,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_204[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -37784,18 +37931,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_204[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -37803,18 +37950,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_204[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -37823,9 +37970,9 @@ _tmp_202_rule(Parser *p) return _res; } -// _tmp_203: ',' | ')' | ':' +// _tmp_205: ',' | ')' | ':' static void * -_tmp_203_rule(Parser *p) +_tmp_205_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37841,18 +37988,18 @@ _tmp_203_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_205[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -37860,18 +38007,18 @@ _tmp_203_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_205[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ':' @@ -37879,18 +38026,18 @@ _tmp_203_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_205[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -37899,9 +38046,9 @@ _tmp_203_rule(Parser *p) return _res; } -// _loop0_205: ',' dotted_name +// _loop0_207: ',' dotted_name static asdl_seq * -_loop0_205_rule(Parser *p) +_loop0_207_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37926,7 +38073,7 @@ _loop0_205_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); + D(fprintf(stderr, "%*c> _loop0_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); Token * _literal; expr_ty elem; while ( @@ -37958,7 +38105,7 @@ _loop0_205_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_205[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_207[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37975,9 +38122,9 @@ _loop0_205_rule(Parser *p) return _seq; } -// _gather_204: dotted_name _loop0_205 +// _gather_206: dotted_name _loop0_207 static asdl_seq * -_gather_204_rule(Parser *p) +_gather_206_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37988,27 +38135,27 @@ _gather_204_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_name _loop0_205 + { // dotted_name _loop0_207 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_205")); + D(fprintf(stderr, "%*c> _gather_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_207")); expr_ty elem; asdl_seq * seq; if ( (elem = dotted_name_rule(p)) // dotted_name && - (seq = _loop0_205_rule(p)) // _loop0_205 + (seq = _loop0_207_rule(p)) // _loop0_207 ) { - D(fprintf(stderr, "%*c+ _gather_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_205")); + D(fprintf(stderr, "%*c+ _gather_206[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_207")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_204[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_205")); + D(fprintf(stderr, "%*c%s _gather_206[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_207")); } _res = NULL; done: @@ -38016,9 +38163,9 @@ _gather_204_rule(Parser *p) return _res; } -// _loop0_207: ',' (expression ['as' star_target]) +// _loop0_209: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_207_rule(Parser *p) +_loop0_209_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38043,13 +38190,13 @@ _loop0_207_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_272_rule(p)) // expression ['as' star_target] + (elem = _tmp_275_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -38075,7 +38222,7 @@ _loop0_207_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_207[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_209[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38092,9 +38239,9 @@ _loop0_207_rule(Parser *p) return _seq; } -// _gather_206: (expression ['as' star_target]) _loop0_207 +// _gather_208: (expression ['as' star_target]) _loop0_209 static asdl_seq * -_gather_206_rule(Parser *p) +_gather_208_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38105,27 +38252,27 @@ _gather_206_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_207 + { // (expression ['as' star_target]) _loop0_209 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); + D(fprintf(stderr, "%*c> _gather_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_209")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_272_rule(p)) // expression ['as' star_target] + (elem = _tmp_275_rule(p)) // expression ['as' star_target] && - (seq = _loop0_207_rule(p)) // _loop0_207 + (seq = _loop0_209_rule(p)) // _loop0_209 ) { - D(fprintf(stderr, "%*c+ _gather_206[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); + D(fprintf(stderr, "%*c+ _gather_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_209")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_206[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); + D(fprintf(stderr, "%*c%s _gather_208[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_209")); } _res = NULL; done: @@ -38133,9 +38280,9 @@ _gather_206_rule(Parser *p) return _res; } -// _loop0_209: ',' (expressions ['as' star_target]) +// _loop0_211: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_209_rule(Parser *p) +_loop0_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38160,13 +38307,13 @@ _loop0_209_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_273_rule(p)) // expressions ['as' star_target] + (elem = _tmp_276_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38192,7 +38339,7 @@ _loop0_209_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_209[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_211[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38209,9 +38356,9 @@ _loop0_209_rule(Parser *p) return _seq; } -// _gather_208: (expressions ['as' star_target]) _loop0_209 +// _gather_210: (expressions ['as' star_target]) _loop0_211 static asdl_seq * -_gather_208_rule(Parser *p) +_gather_210_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38222,27 +38369,27 @@ _gather_208_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_209 + { // (expressions ['as' star_target]) _loop0_211 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); + D(fprintf(stderr, "%*c> _gather_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_211")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_273_rule(p)) // expressions ['as' star_target] + (elem = _tmp_276_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_209_rule(p)) // _loop0_209 + (seq = _loop0_211_rule(p)) // _loop0_211 ) { - D(fprintf(stderr, "%*c+ _gather_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); + D(fprintf(stderr, "%*c+ _gather_210[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_211")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_208[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); + D(fprintf(stderr, "%*c%s _gather_210[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_211")); } _res = NULL; done: @@ -38250,9 +38397,9 @@ _gather_208_rule(Parser *p) return _res; } -// _loop0_211: ',' (expression ['as' star_target]) +// _loop0_213: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_211_rule(Parser *p) +_loop0_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38277,13 +38424,13 @@ _loop0_211_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_274_rule(p)) // expression ['as' star_target] + (elem = _tmp_277_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -38309,7 +38456,7 @@ _loop0_211_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_211[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_213[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38326,9 +38473,9 @@ _loop0_211_rule(Parser *p) return _seq; } -// _gather_210: (expression ['as' star_target]) _loop0_211 +// _gather_212: (expression ['as' star_target]) _loop0_213 static asdl_seq * -_gather_210_rule(Parser *p) +_gather_212_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38339,27 +38486,27 @@ _gather_210_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_211 + { // (expression ['as' star_target]) _loop0_213 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); + D(fprintf(stderr, "%*c> _gather_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_213")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_274_rule(p)) // expression ['as' star_target] + (elem = _tmp_277_rule(p)) // expression ['as' star_target] && - (seq = _loop0_211_rule(p)) // _loop0_211 + (seq = _loop0_213_rule(p)) // _loop0_213 ) { - D(fprintf(stderr, "%*c+ _gather_210[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); + D(fprintf(stderr, "%*c+ _gather_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_213")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_210[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); + D(fprintf(stderr, "%*c%s _gather_212[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_213")); } _res = NULL; done: @@ -38367,9 +38514,9 @@ _gather_210_rule(Parser *p) return _res; } -// _loop0_213: ',' (expressions ['as' star_target]) +// _loop0_215: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_213_rule(Parser *p) +_loop0_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38394,13 +38541,13 @@ _loop0_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_275_rule(p)) // expressions ['as' star_target] + (elem = _tmp_278_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38426,7 +38573,7 @@ _loop0_213_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_215[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38443,9 +38590,9 @@ _loop0_213_rule(Parser *p) return _seq; } -// _gather_212: (expressions ['as' star_target]) _loop0_213 +// _gather_214: (expressions ['as' star_target]) _loop0_215 static asdl_seq * -_gather_212_rule(Parser *p) +_gather_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38456,27 +38603,27 @@ _gather_212_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_213 + { // (expressions ['as' star_target]) _loop0_215 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); + D(fprintf(stderr, "%*c> _gather_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_215")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_275_rule(p)) // expressions ['as' star_target] + (elem = _tmp_278_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_213_rule(p)) // _loop0_213 + (seq = _loop0_215_rule(p)) // _loop0_215 ) { - D(fprintf(stderr, "%*c+ _gather_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); + D(fprintf(stderr, "%*c+ _gather_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_215")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_212[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); + D(fprintf(stderr, "%*c%s _gather_214[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_215")); } _res = NULL; done: @@ -38484,9 +38631,9 @@ _gather_212_rule(Parser *p) return _res; } -// _tmp_214: 'except' | 'finally' +// _tmp_216: 'except' | 'finally' static void * -_tmp_214_rule(Parser *p) +_tmp_216_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38502,18 +38649,18 @@ _tmp_214_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c> _tmp_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 656)) // token='except' ) { - D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c+ _tmp_216[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_216[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); } { // 'finally' @@ -38521,18 +38668,18 @@ _tmp_214_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c> _tmp_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 652)) // token='finally' ) { - D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c+ _tmp_216[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_216[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); } _res = NULL; @@ -38541,9 +38688,9 @@ _tmp_214_rule(Parser *p) return _res; } -// _loop0_215: block +// _loop0_217: block static asdl_seq * -_loop0_215_rule(Parser *p) +_loop0_217_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38568,7 +38715,7 @@ _loop0_215_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38591,7 +38738,7 @@ _loop0_215_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_215[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_217[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38608,9 +38755,9 @@ _loop0_215_rule(Parser *p) return _seq; } -// _loop1_216: except_block +// _loop1_218: except_block static asdl_seq * -_loop1_216_rule(Parser *p) +_loop1_218_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38635,7 +38782,7 @@ _loop1_216_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + D(fprintf(stderr, "%*c> _loop1_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); excepthandler_ty except_block_var; while ( (except_block_var = except_block_rule(p)) // except_block @@ -38658,7 +38805,7 @@ _loop1_216_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_216[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_218[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); } if (_n == 0 || p->error_indicator) { @@ -38680,9 +38827,9 @@ _loop1_216_rule(Parser *p) return _seq; } -// _tmp_217: 'as' NAME +// _tmp_219: 'as' NAME static void * -_tmp_217_rule(Parser *p) +_tmp_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38698,7 +38845,7 @@ _tmp_217_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38707,12 +38854,12 @@ _tmp_217_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_217[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_217[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38721,9 +38868,9 @@ _tmp_217_rule(Parser *p) return _res; } -// _loop0_218: block +// _loop0_220: block static asdl_seq * -_loop0_218_rule(Parser *p) +_loop0_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38748,7 +38895,7 @@ _loop0_218_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -38771,7 +38918,7 @@ _loop0_218_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_218[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_220[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -38788,9 +38935,9 @@ _loop0_218_rule(Parser *p) return _seq; } -// _loop1_219: except_star_block +// _loop1_221: except_star_block static asdl_seq * -_loop1_219_rule(Parser *p) +_loop1_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38815,7 +38962,7 @@ _loop1_219_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + D(fprintf(stderr, "%*c> _loop1_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); excepthandler_ty except_star_block_var; while ( (except_star_block_var = except_star_block_rule(p)) // except_star_block @@ -38838,7 +38985,7 @@ _loop1_219_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_219[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); } if (_n == 0 || p->error_indicator) { @@ -38860,9 +39007,9 @@ _loop1_219_rule(Parser *p) return _seq; } -// _tmp_220: expression ['as' NAME] +// _tmp_222: expression ['as' NAME] static void * -_tmp_220_rule(Parser *p) +_tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38878,22 +39025,22 @@ _tmp_220_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_279_rule(p), !p->error_indicator) // ['as' NAME] ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); } _res = NULL; @@ -38902,9 +39049,9 @@ _tmp_220_rule(Parser *p) return _res; } -// _tmp_221: 'as' NAME +// _tmp_223: 'as' NAME static void * -_tmp_221_rule(Parser *p) +_tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38920,7 +39067,7 @@ _tmp_221_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38929,12 +39076,12 @@ _tmp_221_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38943,9 +39090,9 @@ _tmp_221_rule(Parser *p) return _res; } -// _tmp_222: 'as' NAME +// _tmp_224: 'as' NAME static void * -_tmp_222_rule(Parser *p) +_tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38961,7 +39108,7 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -38970,12 +39117,12 @@ _tmp_222_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -38984,9 +39131,9 @@ _tmp_222_rule(Parser *p) return _res; } -// _tmp_223: NEWLINE | ':' +// _tmp_225: NEWLINE | ':' static void * -_tmp_223_rule(Parser *p) +_tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39002,18 +39149,18 @@ _tmp_223_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } { // ':' @@ -39021,18 +39168,18 @@ _tmp_223_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -39041,9 +39188,9 @@ _tmp_223_rule(Parser *p) return _res; } -// _tmp_224: 'as' NAME +// _tmp_226: 'as' NAME static void * -_tmp_224_rule(Parser *p) +_tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39059,7 +39206,7 @@ _tmp_224_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -39068,12 +39215,12 @@ _tmp_224_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -39082,9 +39229,9 @@ _tmp_224_rule(Parser *p) return _res; } -// _tmp_225: 'as' NAME +// _tmp_227: 'as' NAME static void * -_tmp_225_rule(Parser *p) +_tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39100,7 +39247,7 @@ _tmp_225_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -39109,12 +39256,12 @@ _tmp_225_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -39123,9 +39270,9 @@ _tmp_225_rule(Parser *p) return _res; } -// _tmp_226: positional_patterns ',' +// _tmp_228: positional_patterns ',' static void * -_tmp_226_rule(Parser *p) +_tmp_228_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39141,7 +39288,7 @@ _tmp_226_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -39150,12 +39297,12 @@ _tmp_226_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -39164,9 +39311,9 @@ _tmp_226_rule(Parser *p) return _res; } -// _tmp_227: '->' expression +// _tmp_229: '->' expression static void * -_tmp_227_rule(Parser *p) +_tmp_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39182,7 +39329,7 @@ _tmp_227_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty expression_var; if ( @@ -39191,12 +39338,12 @@ _tmp_227_rule(Parser *p) (expression_var = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = _PyPegen_dummy_name(p, _literal, expression_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -39205,9 +39352,9 @@ _tmp_227_rule(Parser *p) return _res; } -// _tmp_228: '(' arguments? ')' +// _tmp_230: '(' arguments? ')' static void * -_tmp_228_rule(Parser *p) +_tmp_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39223,7 +39370,7 @@ _tmp_228_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39236,12 +39383,12 @@ _tmp_228_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39250,9 +39397,9 @@ _tmp_228_rule(Parser *p) return _res; } -// _tmp_229: '(' arguments? ')' +// _tmp_231: '(' arguments? ')' static void * -_tmp_229_rule(Parser *p) +_tmp_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39268,7 +39415,7 @@ _tmp_229_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39281,12 +39428,12 @@ _tmp_229_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39295,9 +39442,9 @@ _tmp_229_rule(Parser *p) return _res; } -// _loop0_231: ',' double_starred_kvpair +// _loop0_233: ',' double_starred_kvpair static asdl_seq * -_loop0_231_rule(Parser *p) +_loop0_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39322,7 +39469,7 @@ _loop0_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -39354,7 +39501,7 @@ _loop0_231_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39371,9 +39518,9 @@ _loop0_231_rule(Parser *p) return _seq; } -// _gather_230: double_starred_kvpair _loop0_231 +// _gather_232: double_starred_kvpair _loop0_233 static asdl_seq * -_gather_230_rule(Parser *p) +_gather_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39384,27 +39531,27 @@ _gather_230_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_231 + { // double_starred_kvpair _loop0_233 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_231")); + D(fprintf(stderr, "%*c> _gather_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_233")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_231_rule(p)) // _loop0_231 + (seq = _loop0_233_rule(p)) // _loop0_233 ) { - D(fprintf(stderr, "%*c+ _gather_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_231")); + D(fprintf(stderr, "%*c+ _gather_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_233")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_230[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_231")); + D(fprintf(stderr, "%*c%s _gather_232[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_233")); } _res = NULL; done: @@ -39412,9 +39559,9 @@ _gather_230_rule(Parser *p) return _res; } -// _tmp_232: '}' | ',' +// _tmp_234: '}' | ',' static void * -_tmp_232_rule(Parser *p) +_tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39430,18 +39577,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39449,18 +39596,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39469,9 +39616,9 @@ _tmp_232_rule(Parser *p) return _res; } -// _tmp_233: '}' | ',' +// _tmp_235: '}' | ',' static void * -_tmp_233_rule(Parser *p) +_tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39487,18 +39634,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39506,18 +39653,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39526,9 +39673,9 @@ _tmp_233_rule(Parser *p) return _res; } -// _tmp_234: yield_expr | star_expressions +// _tmp_236: yield_expr | star_expressions static void * -_tmp_234_rule(Parser *p) +_tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39544,18 +39691,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39563,18 +39710,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39583,9 +39730,9 @@ _tmp_234_rule(Parser *p) return _res; } -// _tmp_235: yield_expr | star_expressions +// _tmp_237: yield_expr | star_expressions static void * -_tmp_235_rule(Parser *p) +_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39601,18 +39748,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39620,18 +39767,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39640,9 +39787,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: '=' | '!' | ':' | '}' +// _tmp_238: '=' | '!' | ':' | '}' static void * -_tmp_236_rule(Parser *p) +_tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39658,18 +39805,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // '!' @@ -39677,18 +39824,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39696,18 +39843,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39715,18 +39862,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39735,9 +39882,9 @@ _tmp_236_rule(Parser *p) return _res; } -// _tmp_237: yield_expr | star_expressions +// _tmp_239: yield_expr | star_expressions static void * -_tmp_237_rule(Parser *p) +_tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39753,18 +39900,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39772,18 +39919,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39792,9 +39939,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: '!' | ':' | '}' +// _tmp_240: '!' | ':' | '}' static void * -_tmp_238_rule(Parser *p) +_tmp_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39810,18 +39957,18 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39829,18 +39976,18 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39848,18 +39995,18 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39868,9 +40015,9 @@ _tmp_238_rule(Parser *p) return _res; } -// _tmp_239: yield_expr | star_expressions +// _tmp_241: yield_expr | star_expressions static void * -_tmp_239_rule(Parser *p) +_tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39886,18 +40033,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39905,18 +40052,18 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39925,9 +40072,9 @@ _tmp_239_rule(Parser *p) return _res; } -// _tmp_240: yield_expr | star_expressions +// _tmp_242: yield_expr | star_expressions static void * -_tmp_240_rule(Parser *p) +_tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39943,18 +40090,18 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -39962,18 +40109,18 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -39982,9 +40129,9 @@ _tmp_240_rule(Parser *p) return _res; } -// _tmp_241: '!' NAME +// _tmp_243: '!' NAME static void * -_tmp_241_rule(Parser *p) +_tmp_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40000,7 +40147,7 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -40009,12 +40156,12 @@ _tmp_241_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40023,9 +40170,9 @@ _tmp_241_rule(Parser *p) return _res; } -// _tmp_242: ':' | '}' +// _tmp_244: ':' | '}' static void * -_tmp_242_rule(Parser *p) +_tmp_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40041,18 +40188,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -40060,18 +40207,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -40080,9 +40227,9 @@ _tmp_242_rule(Parser *p) return _res; } -// _tmp_243: yield_expr | star_expressions +// _tmp_245: yield_expr | star_expressions static void * -_tmp_243_rule(Parser *p) +_tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40098,18 +40245,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -40117,18 +40264,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -40137,9 +40284,9 @@ _tmp_243_rule(Parser *p) return _res; } -// _tmp_244: '!' NAME +// _tmp_246: '!' NAME static void * -_tmp_244_rule(Parser *p) +_tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40155,7 +40302,7 @@ _tmp_244_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -40164,12 +40311,12 @@ _tmp_244_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40178,9 +40325,9 @@ _tmp_244_rule(Parser *p) return _res; } -// _loop0_245: fstring_format_spec +// _loop0_247: fstring_format_spec static asdl_seq * -_loop0_245_rule(Parser *p) +_loop0_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40205,7 +40352,7 @@ _loop0_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); expr_ty fstring_format_spec_var; while ( (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec @@ -40228,7 +40375,7 @@ _loop0_245_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -40245,9 +40392,9 @@ _loop0_245_rule(Parser *p) return _seq; } -// _tmp_246: yield_expr | star_expressions +// _tmp_248: yield_expr | star_expressions static void * -_tmp_246_rule(Parser *p) +_tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40263,18 +40410,18 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // star_expressions @@ -40282,18 +40429,18 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); expr_ty star_expressions_var; if ( (star_expressions_var = star_expressions_rule(p)) // star_expressions ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); _res = star_expressions_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); } _res = NULL; @@ -40302,9 +40449,9 @@ _tmp_246_rule(Parser *p) return _res; } -// _tmp_247: '!' NAME +// _tmp_249: '!' NAME static void * -_tmp_247_rule(Parser *p) +_tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40320,7 +40467,7 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -40329,12 +40476,12 @@ _tmp_247_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40343,9 +40490,9 @@ _tmp_247_rule(Parser *p) return _res; } -// _tmp_248: ':' | '}' +// _tmp_250: ':' | '}' static void * -_tmp_248_rule(Parser *p) +_tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40361,18 +40508,18 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -40380,18 +40527,18 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -40400,9 +40547,9 @@ _tmp_248_rule(Parser *p) return _res; } -// _tmp_249: '+' | '-' | '*' | '/' | '%' | '//' | '@' +// _tmp_251: '+' | '-' | '*' | '/' | '%' | '//' | '@' static void * -_tmp_249_rule(Parser *p) +_tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40418,18 +40565,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -40437,18 +40584,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '*' @@ -40456,18 +40603,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '/' @@ -40475,18 +40622,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } { // '%' @@ -40494,18 +40641,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 24)) // token='%' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); } { // '//' @@ -40513,18 +40660,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 47)) // token='//' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); } { // '@' @@ -40532,18 +40679,18 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } _res = NULL; @@ -40552,9 +40699,9 @@ _tmp_249_rule(Parser *p) return _res; } -// _tmp_250: '+' | '-' | '~' +// _tmp_252: '+' | '-' | '~' static void * -_tmp_250_rule(Parser *p) +_tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40570,18 +40717,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -40589,18 +40736,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '~' @@ -40608,18 +40755,18 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 31)) // token='~' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); } _res = NULL; @@ -40628,9 +40775,9 @@ _tmp_250_rule(Parser *p) return _res; } -// _tmp_251: star_targets '=' +// _tmp_253: star_targets '=' static void * -_tmp_251_rule(Parser *p) +_tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40646,7 +40793,7 @@ _tmp_251_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -40655,7 +40802,7 @@ _tmp_251_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40665,7 +40812,7 @@ _tmp_251_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40674,9 +40821,9 @@ _tmp_251_rule(Parser *p) return _res; } -// _tmp_252: '.' | '...' +// _tmp_254: '.' | '...' static void * -_tmp_252_rule(Parser *p) +_tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40692,18 +40839,18 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40711,18 +40858,18 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40731,9 +40878,9 @@ _tmp_252_rule(Parser *p) return _res; } -// _tmp_253: '.' | '...' +// _tmp_255: '.' | '...' static void * -_tmp_253_rule(Parser *p) +_tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40749,18 +40896,18 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40768,18 +40915,18 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40788,9 +40935,9 @@ _tmp_253_rule(Parser *p) return _res; } -// _tmp_254: '@' named_expression NEWLINE +// _tmp_256: '@' named_expression NEWLINE static void * -_tmp_254_rule(Parser *p) +_tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40806,7 +40953,7 @@ _tmp_254_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -40818,7 +40965,7 @@ _tmp_254_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40828,7 +40975,7 @@ _tmp_254_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -40837,9 +40984,9 @@ _tmp_254_rule(Parser *p) return _res; } -// _tmp_255: ',' expression +// _tmp_257: ',' expression static void * -_tmp_255_rule(Parser *p) +_tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40855,7 +41002,7 @@ _tmp_255_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty c; if ( @@ -40864,7 +41011,7 @@ _tmp_255_rule(Parser *p) (c = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40874,7 +41021,7 @@ _tmp_255_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -40883,9 +41030,9 @@ _tmp_255_rule(Parser *p) return _res; } -// _tmp_256: ',' star_expression +// _tmp_258: ',' star_expression static void * -_tmp_256_rule(Parser *p) +_tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40901,7 +41048,7 @@ _tmp_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -40910,7 +41057,7 @@ _tmp_256_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40920,7 +41067,7 @@ _tmp_256_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -40929,9 +41076,9 @@ _tmp_256_rule(Parser *p) return _res; } -// _tmp_257: 'or' conjunction +// _tmp_259: 'or' conjunction static void * -_tmp_257_rule(Parser *p) +_tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40947,7 +41094,7 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -40956,7 +41103,7 @@ _tmp_257_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40966,7 +41113,7 @@ _tmp_257_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -40975,9 +41122,9 @@ _tmp_257_rule(Parser *p) return _res; } -// _tmp_258: 'and' inversion +// _tmp_260: 'and' inversion static void * -_tmp_258_rule(Parser *p) +_tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40993,7 +41140,7 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -41002,7 +41149,7 @@ _tmp_258_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41012,7 +41159,7 @@ _tmp_258_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -41021,9 +41168,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: slice | starred_expression +// _tmp_261: slice | starred_expression static void * -_tmp_259_rule(Parser *p) +_tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41039,18 +41186,18 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -41058,18 +41205,18 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -41078,9 +41225,9 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: fstring | string +// _tmp_262: fstring | string static void * -_tmp_260_rule(Parser *p) +_tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41096,18 +41243,18 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); expr_ty fstring_var; if ( (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } { // string @@ -41115,18 +41262,18 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); expr_ty string_var; if ( (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } _res = NULL; @@ -41135,9 +41282,9 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: 'if' disjunction +// _tmp_263: 'if' disjunction static void * -_tmp_261_rule(Parser *p) +_tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41153,7 +41300,7 @@ _tmp_261_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -41162,7 +41309,7 @@ _tmp_261_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41172,7 +41319,7 @@ _tmp_261_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -41181,9 +41328,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: 'if' disjunction +// _tmp_264: 'if' disjunction static void * -_tmp_262_rule(Parser *p) +_tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41199,7 +41346,7 @@ _tmp_262_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -41208,7 +41355,7 @@ _tmp_262_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41218,7 +41365,7 @@ _tmp_262_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -41227,9 +41374,9 @@ _tmp_262_rule(Parser *p) return _res; } -// _loop0_263: (',' bitwise_or) +// _loop0_265: (',' bitwise_or) static asdl_seq * -_loop0_263_rule(Parser *p) +_loop0_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41254,13 +41401,13 @@ _loop0_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); - void *_tmp_277_var; + D(fprintf(stderr, "%*c> _loop0_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_280_var; while ( - (_tmp_277_var = _tmp_277_rule(p)) // ',' bitwise_or + (_tmp_280_var = _tmp_280_rule(p)) // ',' bitwise_or ) { - _res = _tmp_277_var; + _res = _tmp_280_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -41277,7 +41424,7 @@ _loop0_263_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -41294,9 +41441,9 @@ _loop0_263_rule(Parser *p) return _seq; } -// _tmp_264: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_266: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_264_rule(Parser *p) +_tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41312,18 +41459,18 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -41331,20 +41478,20 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_278_var; + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_281_var; if ( - (_tmp_278_var = _tmp_278_rule(p)) // assignment_expression | expression !':=' + (_tmp_281_var = _tmp_281_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_278_var; + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_281_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -41353,9 +41500,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: ',' star_target +// _tmp_267: ',' star_target static void * -_tmp_265_rule(Parser *p) +_tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41371,7 +41518,7 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41380,7 +41527,7 @@ _tmp_265_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41390,7 +41537,7 @@ _tmp_265_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41399,9 +41546,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: ',' star_target +// _tmp_268: ',' star_target static void * -_tmp_266_rule(Parser *p) +_tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41417,7 +41564,7 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41426,7 +41573,7 @@ _tmp_266_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41436,7 +41583,7 @@ _tmp_266_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41445,10 +41592,10 @@ _tmp_266_rule(Parser *p) return _res; } -// _tmp_267: +// _tmp_269: // | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_267_rule(Parser *p) +_tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41464,24 +41611,24 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - asdl_seq * _gather_279_var; + D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + asdl_seq * _gather_282_var; Token * _literal; asdl_seq* kwargs_var; if ( - (_gather_279_var = _gather_279_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (_gather_282_var = _gather_282_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - _res = _PyPegen_dummy_name(p, _gather_279_var, _literal, kwargs_var); + D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + _res = _PyPegen_dummy_name(p, _gather_282_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; @@ -41490,9 +41637,49 @@ _tmp_267_rule(Parser *p) return _res; } -// _tmp_268: star_targets '=' +// _tmp_270: starred_expression !'=' static void * -_tmp_268_rule(Parser *p) +_tmp_270_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // starred_expression !'=' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + expr_ty starred_expression_var; + if ( + (starred_expression_var = starred_expression_rule(p)) // starred_expression + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' + ) + { + D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + _res = starred_expression_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_271: star_targets '=' +static void * +_tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41508,7 +41695,7 @@ _tmp_268_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41517,12 +41704,12 @@ _tmp_268_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41531,9 +41718,9 @@ _tmp_268_rule(Parser *p) return _res; } -// _tmp_269: star_targets '=' +// _tmp_272: star_targets '=' static void * -_tmp_269_rule(Parser *p) +_tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41549,7 +41736,7 @@ _tmp_269_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41558,12 +41745,12 @@ _tmp_269_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41572,9 +41759,9 @@ _tmp_269_rule(Parser *p) return _res; } -// _tmp_270: ')' | '**' +// _tmp_273: ')' | '**' static void * -_tmp_270_rule(Parser *p) +_tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41590,18 +41777,18 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -41609,18 +41796,18 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41629,9 +41816,9 @@ _tmp_270_rule(Parser *p) return _res; } -// _tmp_271: ':' | '**' +// _tmp_274: ':' | '**' static void * -_tmp_271_rule(Parser *p) +_tmp_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41647,18 +41834,18 @@ _tmp_271_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -41666,18 +41853,18 @@ _tmp_271_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41686,9 +41873,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: expression ['as' star_target] +// _tmp_275: expression ['as' star_target] static void * -_tmp_272_rule(Parser *p) +_tmp_275_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41704,22 +41891,22 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_281_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_284_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41728,9 +41915,9 @@ _tmp_272_rule(Parser *p) return _res; } -// _tmp_273: expressions ['as' star_target] +// _tmp_276: expressions ['as' star_target] static void * -_tmp_273_rule(Parser *p) +_tmp_276_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41746,22 +41933,22 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_282_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_285_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41770,9 +41957,9 @@ _tmp_273_rule(Parser *p) return _res; } -// _tmp_274: expression ['as' star_target] +// _tmp_277: expression ['as' star_target] static void * -_tmp_274_rule(Parser *p) +_tmp_277_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41788,22 +41975,22 @@ _tmp_274_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_277[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_283_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_286_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_277[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_277[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41812,9 +41999,9 @@ _tmp_274_rule(Parser *p) return _res; } -// _tmp_275: expressions ['as' star_target] +// _tmp_278: expressions ['as' star_target] static void * -_tmp_275_rule(Parser *p) +_tmp_278_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41830,22 +42017,22 @@ _tmp_275_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_284_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_287_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_278[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_278[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41854,9 +42041,9 @@ _tmp_275_rule(Parser *p) return _res; } -// _tmp_276: 'as' NAME +// _tmp_279: 'as' NAME static void * -_tmp_276_rule(Parser *p) +_tmp_279_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41872,7 +42059,7 @@ _tmp_276_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -41881,12 +42068,12 @@ _tmp_276_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -41895,9 +42082,9 @@ _tmp_276_rule(Parser *p) return _res; } -// _tmp_277: ',' bitwise_or +// _tmp_280: ',' bitwise_or static void * -_tmp_277_rule(Parser *p) +_tmp_280_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41913,7 +42100,7 @@ _tmp_277_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_277[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); Token * _literal; expr_ty bitwise_or_var; if ( @@ -41922,12 +42109,12 @@ _tmp_277_rule(Parser *p) (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or ) { - D(fprintf(stderr, "%*c+ _tmp_277[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_277[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); } _res = NULL; @@ -41936,9 +42123,9 @@ _tmp_277_rule(Parser *p) return _res; } -// _tmp_278: assignment_expression | expression !':=' +// _tmp_281: assignment_expression | expression !':=' static void * -_tmp_278_rule(Parser *p) +_tmp_281_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41954,18 +42141,18 @@ _tmp_278_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_278[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_278[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41973,7 +42160,7 @@ _tmp_278_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41981,12 +42168,12 @@ _tmp_278_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_278[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_278[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -41995,9 +42182,9 @@ _tmp_278_rule(Parser *p) return _res; } -// _loop0_280: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_283: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_280_rule(Parser *p) +_loop0_283_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42022,13 +42209,13 @@ _loop0_280_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_283[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_285_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_288_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -42054,7 +42241,7 @@ _loop0_280_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_280[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_283[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -42071,10 +42258,10 @@ _loop0_280_rule(Parser *p) return _seq; } -// _gather_279: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_280 +// _gather_282: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_283 static asdl_seq * -_gather_279_rule(Parser *p) +_gather_282_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42085,27 +42272,27 @@ _gather_279_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_280 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_283 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_280")); + D(fprintf(stderr, "%*c> _gather_282[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_283")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_285_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_288_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_280_rule(p)) // _loop0_280 + (seq = _loop0_283_rule(p)) // _loop0_283 ) { - D(fprintf(stderr, "%*c+ _gather_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_280")); + D(fprintf(stderr, "%*c+ _gather_282[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_283")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_279[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_280")); + D(fprintf(stderr, "%*c%s _gather_282[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_283")); } _res = NULL; done: @@ -42113,9 +42300,9 @@ _gather_279_rule(Parser *p) return _res; } -// _tmp_281: 'as' star_target +// _tmp_284: 'as' star_target static void * -_tmp_281_rule(Parser *p) +_tmp_284_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42131,7 +42318,7 @@ _tmp_281_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_284[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -42140,12 +42327,12 @@ _tmp_281_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_284[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_284[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -42154,9 +42341,9 @@ _tmp_281_rule(Parser *p) return _res; } -// _tmp_282: 'as' star_target +// _tmp_285: 'as' star_target static void * -_tmp_282_rule(Parser *p) +_tmp_285_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42172,7 +42359,7 @@ _tmp_282_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_282[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_285[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -42181,12 +42368,12 @@ _tmp_282_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_282[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_285[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_282[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_285[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -42195,9 +42382,9 @@ _tmp_282_rule(Parser *p) return _res; } -// _tmp_283: 'as' star_target +// _tmp_286: 'as' star_target static void * -_tmp_283_rule(Parser *p) +_tmp_286_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42213,7 +42400,7 @@ _tmp_283_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_283[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_286[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -42222,12 +42409,12 @@ _tmp_283_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_283[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_286[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_283[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_286[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -42236,9 +42423,9 @@ _tmp_283_rule(Parser *p) return _res; } -// _tmp_284: 'as' star_target +// _tmp_287: 'as' star_target static void * -_tmp_284_rule(Parser *p) +_tmp_287_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42254,7 +42441,7 @@ _tmp_284_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_284[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_287[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -42263,12 +42450,12 @@ _tmp_284_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_284[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_287[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_284[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_287[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -42277,9 +42464,9 @@ _tmp_284_rule(Parser *p) return _res; } -// _tmp_285: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_288: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_285_rule(Parser *p) +_tmp_288_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42295,18 +42482,18 @@ _tmp_285_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_285[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_288[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_285[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_288[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_285[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_288[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -42314,20 +42501,20 @@ _tmp_285_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_285[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_286_var; + D(fprintf(stderr, "%*c> _tmp_288[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_289_var; if ( - (_tmp_286_var = _tmp_286_rule(p)) // assignment_expression | expression !':=' + (_tmp_289_var = _tmp_289_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_285[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_286_var; + D(fprintf(stderr, "%*c+ _tmp_288[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_289_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_285[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_288[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -42336,9 +42523,9 @@ _tmp_285_rule(Parser *p) return _res; } -// _tmp_286: assignment_expression | expression !':=' +// _tmp_289: assignment_expression | expression !':=' static void * -_tmp_286_rule(Parser *p) +_tmp_289_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42354,18 +42541,18 @@ _tmp_286_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_286[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_289[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_286[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_289[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_286[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_289[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -42373,7 +42560,7 @@ _tmp_286_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_286[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_289[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -42381,12 +42568,12 @@ _tmp_286_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_286[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_289[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_286[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_289[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; From c32dc47aca6e8fac152699bc613e015c44ccdba9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 2 Apr 2024 11:59:21 +0100 Subject: [PATCH 046/143] GH-115776: Embed the values array into the object, for "normal" Python objects. (GH-116115) --- Include/cpython/pystats.h | 3 +- Include/internal/pycore_code.h | 5 +- Include/internal/pycore_dict.h | 64 ++- Include/internal/pycore_object.h | 48 +- Include/internal/pycore_opcode_metadata.h | 6 +- Include/internal/pycore_uop_ids.h | 2 +- Include/internal/pycore_uop_metadata.h | 10 +- Include/object.h | 7 +- Lib/test/test_capi/test_mem.py | 4 +- Lib/test/test_class.py | 77 ++- Lib/test/test_opcache.py | 13 +- ...-02-24-03-39-09.gh-issue-115776.THJXqg.rst | 4 + Modules/_testbuffer.c | 3 + Modules/_testcapimodule.c | 4 +- Modules/_testinternalcapi.c | 20 +- Objects/dictobject.c | 464 +++++++++--------- Objects/object.c | 100 ++-- Objects/object_layout.md | 91 +++- Objects/object_layout_312.gv | 1 + Objects/object_layout_312.png | Bin 30688 -> 33040 bytes Objects/object_layout_313.gv | 45 ++ Objects/object_layout_313.png | Bin 0 -> 35055 bytes Objects/object_layout_full_313.gv | 25 + Objects/object_layout_full_313.png | Bin 0 -> 16983 bytes Objects/typeobject.c | 84 ++-- Python/bytecodes.c | 52 +- Python/executor_cases.c.h | 44 +- Python/gc.c | 7 +- Python/gc_free_threading.c | 6 +- Python/generated_cases.c.h | 54 +- Python/optimizer_cases.c.h | 2 +- Python/specialize.c | 51 +- Tools/cases_generator/analyzer.py | 7 +- Tools/gdb/libpython.py | 14 +- Tools/scripts/summarize_stats.py | 5 +- 35 files changed, 786 insertions(+), 536 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst create mode 100644 Objects/object_layout_313.gv create mode 100644 Objects/object_layout_313.png create mode 100644 Objects/object_layout_full_313.gv create mode 100644 Objects/object_layout_full_313.png diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 2fb7723f583cc7..e74fdd4d32e26c 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -77,12 +77,11 @@ typedef struct _object_stats { uint64_t frees; uint64_t to_freelist; uint64_t from_freelist; - uint64_t new_values; + uint64_t inline_values; uint64_t dict_materialized_on_request; uint64_t dict_materialized_new_key; uint64_t dict_materialized_too_big; uint64_t dict_materialized_str_subclass; - uint64_t dict_dematerialized; uint64_t type_cache_hits; uint64_t type_cache_misses; uint64_t type_cache_dunder_hits; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index e004783ee48198..6c90c9e284103c 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -79,7 +79,10 @@ typedef struct { typedef struct { uint16_t counter; uint16_t type_version[2]; - uint16_t keys_version[2]; + union { + uint16_t keys_version[2]; + uint16_t dict_offset; + }; uint16_t descr[4]; } _PyLoadMethodCache; diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index ef59960dbab071..5507bdd539cc42 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -11,7 +11,7 @@ extern "C" { #include "pycore_freelist.h" // _PyFreeListState #include "pycore_identifier.h" // _Py_Identifier -#include "pycore_object.h" // PyDictOrValues +#include "pycore_object.h" // PyManagedDictPointer // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); @@ -181,6 +181,10 @@ struct _dictkeysobject { * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order. */ struct _dictvalues { + uint8_t capacity; + uint8_t size; + uint8_t embedded; + uint8_t valid; PyObject *values[1]; }; @@ -196,6 +200,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) { size_t index = (size_t)1 << dk->dk_log2_index_bytes; return (&indices[index]); } + static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) { assert(dk->dk_kind == DICT_KEYS_GENERAL); return (PyDictKeyEntry*)_DK_ENTRIES(dk); @@ -211,9 +216,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) -#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1] -#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2] - #ifdef Py_GIL_DISABLED #define DICT_NEXT_VERSION(INTERP) \ (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) @@ -246,25 +248,63 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK); } -extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); -PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); +extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj); + PyAPI_FUNC(PyObject *)_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, Py_ssize_t length); +static inline uint8_t * +get_insertion_order_array(PyDictValues *values) +{ + return (uint8_t *)&values->values[values->capacity]; +} + static inline void _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) { assert(ix < SHARED_KEYS_MAX_SIZE); - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; - assert(size+2 < DICT_VALUES_SIZE(values)); - size++; - size_ptr[-size] = (uint8_t)ix; - *size_ptr = size; + int size = values->size; + uint8_t *array = get_insertion_order_array(values); + assert(size < values->capacity); + assert(((uint8_t)ix) == ix); + array[size] = (uint8_t)ix; + values->size = size+1; +} + +static inline size_t +shared_keys_usable_size(PyDictKeysObject *keys) +{ +#ifdef Py_GIL_DISABLED + // dk_usable will decrease for each instance that is created and each + // value that is added. dk_nentries will increase for each value that + // is added. We want to always return the right value or larger. + // We therefore increase dk_nentries first and we decrease dk_usable + // second, and conversely here we read dk_usable first and dk_entries + // second (to avoid the case where we read entries before the increment + // and read usable after the decrement) + return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) + + _Py_atomic_load_ssize_acquire(&keys->dk_nentries)); +#else + return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; +#endif } +static inline size_t +_PyInlineValuesSize(PyTypeObject *tp) +{ + PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys; + assert(keys != NULL); + size_t size = shared_keys_usable_size(keys); + size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *)); + assert(prefix_size < 256); + return prefix_size + (size + 1) * sizeof(PyObject *); +} + +int +_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 0b17ddf0c973ef..4fc5e9bf653c1c 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -265,9 +265,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) { assert(op != NULL); Py_SET_TYPE(op, typeobj); - if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) { - Py_INCREF(typeobj); - } + assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); + Py_INCREF(typeobj); _Py_NewReference(op); } @@ -611,8 +610,7 @@ extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); -extern int _PyObject_InitializeDict(PyObject *obj); -int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value); PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, @@ -627,46 +625,26 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, #endif typedef union { - PyObject *dict; - /* Use a char* to generate a warning if directly assigning a PyDictValues */ - char *values; -} PyDictOrValues; + PyDictObject *dict; +} PyManagedDictPointer; -static inline PyDictOrValues * -_PyObject_DictOrValuesPointer(PyObject *obj) +static inline PyManagedDictPointer * +_PyObject_ManagedDictPointer(PyObject *obj) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET); -} - -static inline int -_PyDictOrValues_IsValues(PyDictOrValues dorv) -{ - return ((uintptr_t)dorv.values) & 1; + return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET); } static inline PyDictValues * -_PyDictOrValues_GetValues(PyDictOrValues dorv) -{ - assert(_PyDictOrValues_IsValues(dorv)); - return (PyDictValues *)(dorv.values + 1); -} - -static inline PyObject * -_PyDictOrValues_GetDict(PyDictOrValues dorv) +_PyObject_InlineValues(PyObject *obj) { - assert(!_PyDictOrValues_IsValues(dorv)); - return dorv.dict; -} - -static inline void -_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) -{ - ptr->values = ((char *)values) - 1; + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject)); + return (PyDictValues *)((char *)obj + sizeof(PyObject)); } extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); -extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); // Export for 'math' shared extension diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index de525f72d3523e..aa87dc413876f0 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1085,7 +1085,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1269,7 +1269,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, @@ -1316,7 +1316,7 @@ _PyOpcode_macro_expansion[256] = { [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } }, [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index bcb10ab723ecba..54dc6dcf408116 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -109,7 +109,7 @@ extern "C" { #define _GUARD_BOTH_INT 347 #define _GUARD_BOTH_UNICODE 348 #define _GUARD_BUILTINS_VERSION 349 -#define _GUARD_DORV_VALUES 350 +#define _GUARD_DORV_NO_DICT 350 #define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351 #define _GUARD_GLOBALS_VERSION 352 #define _GUARD_IS_FALSE_POP 353 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 51206cd4ca2fdf..0f2046fb3d0c3d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -136,8 +136,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, @@ -145,7 +145,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_CLASS_0] = 0, [_LOAD_ATTR_CLASS_1] = 0, [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, - [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = 0, [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -342,7 +342,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", - [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", + [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", @@ -736,7 +736,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_CLASS: return 1; - case _GUARD_DORV_VALUES: + case _GUARD_DORV_NO_DICT: return 1; case _STORE_ATTR_INSTANCE_VALUE: return 2; diff --git a/Include/object.h b/Include/object.h index 96790844a7b9f0..13443329dfb5a2 100644 --- a/Include/object.h +++ b/Include/object.h @@ -629,13 +629,18 @@ given type object has a specified feature. /* Track types initialized using _PyStaticType_InitBuiltin(). */ #define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1) +/* The values array is placed inline directly after the rest of + * the object. Implies Py_TPFLAGS_HAVE_GC. + */ +#define Py_TPFLAGS_INLINE_VALUES (1 << 2) + /* Placement of weakref pointers are managed by the VM, not by the type. * The VM will automatically set tp_weaklistoffset. */ #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3) /* Placement of dict (and values) pointers are managed by the VM, not by the type. - * The VM will automatically set tp_dictoffset. + * The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC. */ #define Py_TPFLAGS_MANAGED_DICT (1 << 4) diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 04f17a9ec9e72a..1958ecc0df4a7c 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -148,8 +148,8 @@ class C(): pass self.assertIn(b'MemoryError', out) *_, count = line.split(b' ') count = int(count) - self.assertLessEqual(count, i*5) - self.assertGreaterEqual(count, i*5-2) + self.assertLessEqual(count, i*10) + self.assertGreaterEqual(count, i*10-4) # Py_GIL_DISABLED requires mimalloc (not malloc) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 0cf06243dc8da0..4c1814142736e3 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -2,7 +2,6 @@ import unittest - testmeths = [ # Binary operations @@ -789,5 +788,81 @@ def __init__(self, obj): self.assertEqual(calls, 100) +from _testinternalcapi import has_inline_values + +Py_TPFLAGS_MANAGED_DICT = (1 << 2) + +class Plain: + pass + + +class WithAttrs: + + def __init__(self): + self.a = 1 + self.b = 2 + self.c = 3 + self.d = 4 + + +class TestInlineValues(unittest.TestCase): + + def test_flags(self): + self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + + def test_has_inline_values(self): + c = Plain() + self.assertTrue(has_inline_values(c)) + del c.__dict__ + self.assertFalse(has_inline_values(c)) + + def test_instances(self): + self.assertTrue(has_inline_values(Plain())) + self.assertTrue(has_inline_values(WithAttrs())) + + def test_inspect_dict(self): + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__ + self.assertTrue(has_inline_values(c)) + + def test_update_dict(self): + d = { "e": 5, "f": 6 } + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__.update(d) + self.assertTrue(has_inline_values(c)) + + @staticmethod + def set_100(obj): + for i in range(100): + setattr(obj, f"a{i}", i) + + def check_100(self, obj): + for i in range(100): + self.assertEqual(getattr(obj, f"a{i}"), i) + + def test_many_attributes(self): + class C: pass + c = C() + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + c = C() + self.assertTrue(has_inline_values(c)) + + def test_many_attributes_with_dict(self): + class C: pass + c = C() + d = c.__dict__ + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5fb4b815c95d07..8829c9a6d88261 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1044,20 +1044,13 @@ def test_dict_materialization(self): c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) + self.assertEqual(c.__dict__, {"a":1, "b": 2}) def test_dict_dematerialization(self): c = C() c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) for _ in range(100): c.a self.assertEqual( @@ -1072,10 +1065,6 @@ def test_dict_dematerialization_multiple_refs(self): d = c.__dict__ for _ in range(100): c.a - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) self.assertIs(c.__dict__, d) def test_dict_dematerialization_copy(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst new file mode 100644 index 00000000000000..5974b1882acb22 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst @@ -0,0 +1,4 @@ +The array of values, the ``PyDictValues`` struct is now embedded in the +object during allocation. This provides better performance in the common +case, and does not degrade as much when the object's ``__dict__`` is +materialized. diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 5084bcadb10f85..cad21bdb4d85ed 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -2820,6 +2820,9 @@ static int _testbuffer_exec(PyObject *mod) { Py_SET_TYPE(&NDArray_Type, &PyType_Type); + if (PyType_Ready(&NDArray_Type)) { + return -1; + } if (PyModule_AddType(mod, &NDArray_Type) < 0) { return -1; } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index e9db6e5683e344..b2af47d05ee196 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3869,7 +3869,9 @@ PyInit__testcapi(void) return NULL; Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type); - + if (PyType_Ready(&_HashInheritanceTester_Type) < 0) { + return NULL; + } if (PyType_Ready(&matmulType) < 0) return NULL; Py_INCREF(&matmulType); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c07652facc0ae2..d6d50e75b612df 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -15,7 +15,7 @@ #include "pycore_ceval.h" // _PyEval_AddPendingCall() #include "pycore_compile.h" // _PyCompile_CodeGen() #include "pycore_context.h" // _PyContext_NewHamtForTests() -#include "pycore_dict.h" // _PyDictOrValues_GetValues() +#include "pycore_dict.h" // _PyManagedDictPointer_GetValues() #include "pycore_fileutils.h" // _Py_normpath() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head @@ -1297,14 +1297,13 @@ static PyObject * get_object_dict_values(PyObject *self, PyObject *obj) { PyTypeObject *type = Py_TYPE(obj); - if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) { + if (!_PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) { Py_RETURN_NONE; } - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (!_PyDictOrValues_IsValues(dorv)) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (!values->valid) { Py_RETURN_NONE; } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(keys != NULL); int size = (int)keys->dk_nentries; @@ -1784,6 +1783,16 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } #endif +static PyObject * +has_inline_values(PyObject *self, PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_InlineValues(obj)->valid) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1857,6 +1866,7 @@ static PyMethodDef module_functions[] = { _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, + {"has_inline_values", has_inline_values, METH_O}, #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 536746ca41eed5..58a3d979339c49 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -361,6 +361,10 @@ static int dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, PyObject **result, int incref_result); +#ifndef NDEBUG +static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); +#endif + #include "clinic/dictobject.c.h" @@ -624,8 +628,9 @@ static inline int get_index_from_order(PyDictObject *mp, Py_ssize_t i) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); - assert(i < (((char *)mp->ma_values)[-2])); - return ((char *)mp->ma_values)[-3-i]; + assert(i < mp->ma_values->size); + uint8_t *array = get_insertion_order_array(mp->ma_values); + return array[i]; } #ifdef DEBUG_PYDICT @@ -672,6 +677,10 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) else { CHECK(keys->dk_kind == DICT_KEYS_SPLIT); CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + if (mp->ma_values->embedded) { + CHECK(mp->ma_values->embedded == 1); + CHECK(mp->ma_values->valid == 1); + } } if (check_content) { @@ -821,33 +830,44 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) PyMem_Free(keys); } +static size_t +values_size_from_count(size_t count) +{ + assert(count >= 1); + size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *)); + assert(suffix_size < 128); + assert(suffix_size % sizeof(PyObject *) == 0); + return (count + 1) * sizeof(PyObject *) + suffix_size; +} + +#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) + static inline PyDictValues* new_values(size_t size) { - assert(size >= 1); - size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *)); - assert(prefix_size < 256); - size_t n = prefix_size + size * sizeof(PyObject *); - uint8_t *mem = PyMem_Malloc(n); - if (mem == NULL) { + size_t n = values_size_from_count(size); + PyDictValues *res = (PyDictValues *)PyMem_Malloc(n); + if (res == NULL) { return NULL; } - assert(prefix_size % sizeof(PyObject *) == 0); - mem[prefix_size-1] = (uint8_t)prefix_size; - return (PyDictValues*)(mem + prefix_size); + res->embedded = 0; + res->size = 0; + assert(size < 256); + res->capacity = (uint8_t)size; + return res; } static inline void free_values(PyDictValues *values, bool use_qsbr) { - int prefix_size = DICT_VALUES_SIZE(values); + assert(values->embedded == 0); #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(((char *)values)-prefix_size); + _PyMem_FreeDelayed(values); return; } #endif - PyMem_Free(((char *)values)-prefix_size); + PyMem_Free(values); } /* Consumes a reference to the keys object */ @@ -887,24 +907,6 @@ new_dict(PyInterpreterState *interp, return (PyObject *)mp; } -static inline size_t -shared_keys_usable_size(PyDictKeysObject *keys) -{ -#ifdef Py_GIL_DISABLED - // dk_usable will decrease for each instance that is created and each - // value that is added. dk_nentries will increase for each value that - // is added. We want to always return the right value or larger. - // We therefore increase dk_nentries first and we decrease dk_usable - // second, and conversely here we read dk_usable first and dk_entries - // second (to avoid the case where we read entries before the increment - // and read usable after the decrement) - return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) + - _Py_atomic_load_ssize_acquire(&keys->dk_nentries)); -#else - return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; -#endif -} - /* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) @@ -915,7 +917,6 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) dictkeys_decref(interp, keys, false); return PyErr_NoMemory(); } - ((char *)values)[-2] = 0; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } @@ -1419,7 +1420,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb if (values == NULL) goto read_failed; - uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values)); + uint8_t capacity = _Py_atomic_load_uint8_relaxed(&values->capacity); if (ix >= (Py_ssize_t)capacity) goto read_failed; @@ -1525,6 +1526,7 @@ _PyDict_MaybeUntrack(PyObject *op) return; mp = (PyDictObject *) op; + ASSERT_CONSISTENT(mp); numentries = mp->ma_keys->dk_nentries; if (_PyDict_HasSplitTable(mp)) { for (i = 0; i < numentries; i++) { @@ -1945,7 +1947,14 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, set_keys(mp, newkeys); dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); set_values(mp, NULL); - free_values(oldvalues, IS_DICT_SHARED(mp)); + if (oldvalues->embedded) { + assert(oldvalues->embedded == 1); + assert(oldvalues->valid == 1); + oldvalues->valid = 0; + } + else { + free_values(oldvalues, IS_DICT_SHARED(mp)); + } } else { // oldkeys is combined. if (oldkeys->dk_kind == DICT_KEYS_GENERAL) { @@ -2464,17 +2473,19 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, static void delete_index_from_values(PyDictValues *values, Py_ssize_t ix) { - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; + uint8_t *array = get_insertion_order_array(values); + int size = values->size; + assert(size <= values->capacity); int i; - for (i = 1; size_ptr[-i] != ix; i++) { - assert(i <= size); + for (i = 0; array[i] != ix; i++) { + assert(i < size); } - assert(i <= size); + assert(i < size); + size--; for (; i < size; i++) { - size_ptr[-i] = size_ptr[-i-1]; + array[i] = array[i+1]; } - *size_ptr = size -1; + values->size = size; } static int @@ -2669,10 +2680,12 @@ clear_lock_held(PyObject *op) mp->ma_version_tag = new_version; /* ...then clear the keys and values */ if (oldvalues != NULL) { - n = oldkeys->dk_nentries; - for (i = 0; i < n; i++) - Py_CLEAR(oldvalues->values[i]); - free_values(oldvalues, IS_DICT_SHARED(mp)); + if (!oldvalues->embedded) { + n = oldkeys->dk_nentries; + for (i = 0; i < n; i++) + Py_CLEAR(oldvalues->values[i]); + free_values(oldvalues, IS_DICT_SHARED(mp)); + } dictkeys_decref(interp, oldkeys, false); } else { @@ -3059,10 +3072,12 @@ dict_dealloc(PyObject *self) PyObject_GC_UnTrack(mp); Py_TRASHCAN_BEGIN(mp, dict_dealloc) if (values != NULL) { - for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { - Py_XDECREF(values->values[i]); + if (values->embedded == 0) { + for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { + Py_XDECREF(values->values[i]); + } + free_values(values, false); } - free_values(values, false); dictkeys_decref(interp, keys, false); } else if (keys != NULL) { @@ -3595,10 +3610,12 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe PyDictKeysObject *okeys = other->ma_keys; // If other is clean, combined, and just allocated, just clone it. - if (other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { + if (mp->ma_values == NULL && + other->ma_values == NULL && + other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used) + ) { uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); @@ -3608,11 +3625,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe ensure_shared_on_resize(mp); dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; - if (_PyDict_HasSplitTable(mp)) { - free_values(mp->ma_values, IS_DICT_SHARED(mp)); - mp->ma_values = NULL; - } - mp->ma_used = other->ma_used; mp->ma_version_tag = new_version; ASSERT_CONSISTENT(mp); @@ -3816,6 +3828,27 @@ dict_copy_impl(PyDictObject *self) return PyDict_Copy((PyObject *)self); } +/* Copies the values, but does not change the reference + * counts of the objects in the array. */ +static PyDictValues * +copy_values(PyDictValues *values) +{ + PyDictValues *newvalues = new_values(values->capacity); + if (newvalues == NULL) { + PyErr_NoMemory(); + return NULL; + } + newvalues->size = values->size; + uint8_t *values_order = get_insertion_order_array(values); + uint8_t *new_values_order = get_insertion_order_array(newvalues); + memcpy(new_values_order, values_order, values->capacity); + for (int i = 0; i < values->capacity; i++) { + newvalues->values[i] = values->values[i]; + } + assert(newvalues->embedded == 0); + return newvalues; +} + static PyObject * copy_lock_held(PyObject *o) { @@ -3833,26 +3866,23 @@ copy_lock_held(PyObject *o) if (_PyDict_HasSplitTable(mp)) { PyDictObject *split_copy; - size_t size = shared_keys_usable_size(mp->ma_keys); - PyDictValues *newvalues = new_values(size); - if (newvalues == NULL) + PyDictValues *newvalues = copy_values(mp->ma_values); + if (newvalues == NULL) { return PyErr_NoMemory(); + } split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); if (split_copy == NULL) { free_values(newvalues, false); return NULL; } - size_t prefix_size = ((uint8_t *)newvalues)[-1]; - memcpy(((char *)newvalues)-prefix_size, ((char *)mp->ma_values)-prefix_size, prefix_size-1); + for (size_t i = 0; i < newvalues->capacity; i++) { + Py_XINCREF(newvalues->values[i]); + } split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); dictkeys_incref(mp->ma_keys); - for (size_t i = 0; i < size; i++) { - PyObject *value = mp->ma_values->values[i]; - split_copy->ma_values->values[i] = Py_XNewRef(value); - } if (_PyObject_GC_IS_TRACKED(mp)) _PyObject_GC_TRACK(split_copy); return (PyObject *)split_copy; @@ -4406,8 +4436,10 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); + if (!mp->ma_values->embedded) { + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); + } } } else { @@ -5296,12 +5328,6 @@ acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, return 0; } -static Py_ssize_t -load_values_used_size(PyDictValues *values) -{ - return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values)); -} - static int dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, PyObject **out_key, PyObject **out_value) @@ -5330,7 +5356,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, goto concurrent_modification; } - Py_ssize_t used = load_values_used_size(values); + Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size); if (i >= used) { goto fail; } @@ -6539,15 +6565,15 @@ _PyDict_NewKeysForClass(void) return keys; } -#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) - -int +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); + OBJECT_STAT_INC(inline_values); #ifdef Py_GIL_DISABLED Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable); if (usable > 1) { @@ -6563,49 +6589,19 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) } #endif size_t size = shared_keys_usable_size(keys); - PyDictValues *values = new_values(size); - if (values == NULL) { - PyErr_NoMemory(); - return -1; - } - assert(((uint8_t *)values)[-1] >= (size + 2)); - ((uint8_t *)values)[-2] = 0; + PyDictValues *values = _PyObject_InlineValues(obj); + assert(size < 256); + values->capacity = (uint8_t)size; + values->size = 0; + values->embedded = 1; + values->valid = 1; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values); - return 0; -} - -int -_PyObject_InitializeDict(PyObject *obj) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyTypeObject *tp = Py_TYPE(obj); - if (tp->tp_dictoffset == 0) { - return 0; - } - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - OBJECT_STAT_INC(new_values); - return _PyObject_InitInlineValues(obj, tp); - } - PyObject *dict; - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - } - else { - dict = PyDict_New(); - } - if (dict == NULL) { - return -1; - } - PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - *dictptr = dict; - return 0; + _PyObject_ManagedDictPointer(obj)->dict = NULL; } -static PyObject * +static PyDictObject * make_dict_from_instance_attributes(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values) { @@ -6620,56 +6616,24 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(interp, keys, values, used, 0); + PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); if (track && res) { _PyObject_GC_TRACK(res); } return res; } -PyObject * -_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) +PyDictObject * +_PyObject_MakeDictFromInstanceAttributes(PyObject *obj) { + PyDictValues *values = _PyObject_InlineValues(obj); PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); return make_dict_from_instance_attributes(interp, keys, values); } -// Return true if the dict was dematerialized, false otherwise. -bool -_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) -{ - assert(_PyObject_DictOrValuesPointer(obj) == dorv); - assert(!_PyDictOrValues_IsValues(*dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); - if (dict == NULL) { - return false; - } - // It's likely that this dict still shares its keys (if it was materialized - // on request and not heavily modified): - if (!PyDict_CheckExact(dict)) { - return false; - } - assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); - if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || - !has_unique_reference((PyObject *)dict)) - { - return false; - } - ensure_shared_on_resize(dict); - assert(dict->ma_values); - // We have an opportunity to do something *really* cool: dematerialize it! - _PyDictKeys_DecRef(dict->ma_keys); - _PyDictOrValues_SetValues(dorv, dict->ma_values); - OBJECT_STAT_INC(dict_dematerialized); - // Don't try this at home, kids: - dict->ma_keys = NULL; - dict->ma_values = NULL; - Py_DECREF(dict); - return true; -} int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, @@ -6679,7 +6643,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); - assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); Py_ssize_t ix = DKIX_EMPTY; if (PyUnicode_CheckExact(name)) { Py_hash_t hash = unicode_get_hash(name); @@ -6717,18 +6681,21 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, } #endif } + PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; if (ix == DKIX_EMPTY) { - PyObject *dict = make_dict_from_instance_attributes( - interp, keys, values); if (dict == NULL) { - return -1; + dict = make_dict_from_instance_attributes( + interp, keys, values); + if (dict == NULL) { + return -1; + } + _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict; } - _PyObject_DictOrValuesPointer(obj)->dict = dict; if (value == NULL) { - return PyDict_DelItem(dict, name); + return PyDict_DelItem((PyObject *)dict, name); } else { - return PyDict_SetItem(dict, name, value); + return PyDict_SetItem((PyObject *)dict, name, value); } } PyObject *old_value = values->values[ix]; @@ -6741,10 +6708,18 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, return -1; } _PyDictValues_AddToInsertionOrder(values, ix); + if (dict) { + assert(dict->ma_values == values); + dict->ma_used++; + } } else { if (value == NULL) { delete_index_from_values(values, ix); + if (dict) { + assert(dict->ma_values == values); + dict->ma_used--; + } } Py_DECREF(old_value); } @@ -6760,9 +6735,9 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) { PyTypeObject *tp = Py_TYPE(obj); CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + if (_PyManagedDictPointer_IsValues(*managed_dict)) { + PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict); int size = ((uint8_t *)values)[-2]; int count = 0; PyDictKeysObject *keys = CACHED_KEYS(tp); @@ -6774,8 +6749,8 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) CHECK(size == count); } else { - if (dorv_ptr->dict != NULL) { - CHECK(PyDict_Check(dorv_ptr->dict)); + if (managed_dict->dict != NULL) { + CHECK(PyDict_Check(managed_dict->dict)); } } return 1; @@ -6804,23 +6779,27 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) if (tp->tp_dictoffset == 0) { return 1; } - PyObject *dict; - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { + PyDictObject *dict; + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { PyDictKeysObject *keys = CACHED_KEYS(tp); for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) { + if (values->values[i] != NULL) { return 0; } } return 1; } - dict = _PyDictOrValues_GetDict(dorv); + dict = _PyObject_ManagedDictPointer(obj)->dict; + } + else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); + dict = managed_dict->dict; } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - dict = *dictptr; + dict = (PyDictObject *)*dictptr; } if (dict == NULL) { return 1; @@ -6828,23 +6807,6 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) return ((PyDictObject *)dict)->ma_used == 0; } -void -_PyObject_FreeInstanceAttributes(PyObject *self) -{ - PyTypeObject *tp = Py_TYPE(self); - assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - if (!_PyDictOrValues_IsValues(dorv)) { - return; - } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_XDECREF(values->values[i]); - } - free_values(values, IS_DICT_SHARED((PyDictObject*)self)); -} - int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) { @@ -6852,74 +6814,101 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return 0; } - assert(tp->tp_dictoffset); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_VISIT(values->values[i]); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_VISIT(values->values[i]); + } + return 0; } } - else { - PyObject *dict = _PyDictOrValues_GetDict(dorv); - Py_VISIT(dict); - } + Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict); return 0; } void PyObject_ClearManagedDict(PyObject *obj) { + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); PyTypeObject *tp = Py_TYPE(obj); - if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - return; - } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_CLEAR(values->values[i]); - } - dorv_ptr->dict = NULL; - free_values(values, IS_DICT_SHARED((PyDictObject*)obj)); - } - else { - PyObject *dict = dorv_ptr->dict; + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; if (dict) { - dorv_ptr->dict = NULL; + _PyDict_DetachFromObject(dict, obj); + _PyObject_ManagedDictPointer(obj)->dict = NULL; Py_DECREF(dict); } + else { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_CLEAR(values->values[i]); + } + values->valid = 0; + } + } + } + else { + Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict); + } + assert(_PyObject_InlineValuesConsistencyCheck(obj)); +} + +int +_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) +{ + assert(_PyObject_ManagedDictPointer(obj)->dict == mp); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + if (mp->ma_values == NULL || mp->ma_values != _PyObject_InlineValues(obj)) { + return 0; + } + assert(mp->ma_values->embedded == 1); + assert(mp->ma_values->valid == 1); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + Py_BEGIN_CRITICAL_SECTION(mp); + mp->ma_values = copy_values(mp->ma_values); + _PyObject_InlineValues(obj)->valid = 0; + Py_END_CRITICAL_SECTION(); + if (mp->ma_values == NULL) { + return -1; } + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + ASSERT_CONSISTENT(mp); + return 0; } PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { - PyObject *dict; PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + PyDictObject *dict = managed_dict->dict; + if (dict == NULL && + (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_InlineValues(obj)->valid + ) { + PyDictValues *values = _PyObject_InlineValues(obj); OBJECT_STAT_INC(dict_materialized_on_request); dict = make_dict_from_instance_attributes( interp, CACHED_KEYS(tp), values); if (dict != NULL) { - dorv_ptr->dict = dict; + managed_dict->dict = (PyDictObject *)dict; } } else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); + dict = managed_dict->dict; if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); OBJECT_STAT_INC(dict_materialized_on_request); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - dorv_ptr->dict = dict; + dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); + managed_dict->dict = (PyDictObject *)dict; } } + return Py_XNewRef((PyObject *)dict); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -6928,7 +6917,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) "This object has no __dict__"); return NULL; } - dict = *dictptr; + PyObject *dict = *dictptr; if (dict == NULL) { PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { @@ -6940,8 +6929,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context) *dictptr = dict = PyDict_New(); } } + return Py_XNewRef(dict); } - return Py_XNewRef(dict); } int @@ -6958,7 +6947,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, assert(dictptr != NULL); dict = *dictptr; if (dict == NULL) { - assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)); + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); dictkeys_incref(cached); dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) @@ -7118,3 +7107,24 @@ _PyDict_SendEvent(int watcher_bits, watcher_bits >>= 1; } } + +#ifndef NDEBUG +static int +_PyObject_InlineValuesConsistencyCheck(PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + return 1; + } + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = (PyDictObject *)_PyObject_ManagedDictPointer(obj)->dict; + if (dict == NULL) { + return 1; + } + if (dict->ma_values == _PyObject_InlineValues(obj) || + _PyObject_InlineValues(obj)->valid == 0) { + return 1; + } + assert(0); + return 0; +} +#endif diff --git a/Objects/object.c b/Objects/object.c index b4f0fd4d7db941..60642d899bcafa 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1396,16 +1396,16 @@ _PyObject_GetDictPtr(PyObject *obj) if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return _PyObject_ComputedDictPointer(obj); } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, _PyDictOrValues_GetValues(*dorv_ptr)); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + if (managed_dict->dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(obj); if (dict == NULL) { PyErr_Clear(); return NULL; } - dorv_ptr->dict = dict; + managed_dict->dict = dict; } - return &dorv_ptr->dict; + return (PyObject **)&managed_dict->dict; } PyObject * @@ -1474,21 +1474,19 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } PyObject *dict; - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); - if (attr != NULL) { - *method = attr; - Py_XDECREF(descr); - return 0; - } - dict = NULL; - } - else { - dict = dorv_ptr->dict; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) { + PyDictValues *values = _PyObject_InlineValues(obj); + PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); + if (attr != NULL) { + *method = attr; + Py_XDECREF(descr); + return 0; } + dict = NULL; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); + dict = (PyObject *)managed_dict->dict; } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -1581,29 +1579,27 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } } if (dict == NULL) { - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - if (PyUnicode_CheckExact(name)) { - res = _PyObject_GetInstanceAttribute(obj, values, name); - if (res != NULL) { - goto done; - } - } - else { - dict = _PyObject_MakeDictFromInstanceAttributes(obj, values); - if (dict == NULL) { - res = NULL; - goto done; - } - dorv_ptr->dict = dict; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (PyUnicode_CheckExact(name)) { + res = _PyObject_GetInstanceAttribute(obj, values, name); + if (res != NULL) { + goto done; } } else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); + dict = (PyObject *)_PyObject_MakeDictFromInstanceAttributes(obj); + if (dict == NULL) { + res = NULL; + goto done; + } + _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict; } } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); + dict = (PyObject *)managed_dict->dict; + } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); if (dictptr) { @@ -1697,22 +1693,14 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, if (dict == NULL) { PyObject **dictptr; - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } - dictptr = &dorv_ptr->dict; - if (*dictptr == NULL) { - if (_PyObject_InitInlineValues(obj, tp) < 0) { - goto done; - } - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) { + res = _PyObject_StoreInstanceAttribute( + obj, _PyObject_InlineValues(obj), name, value); + goto error_check; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + dictptr = (PyObject **)&managed_dict->dict; } else { dictptr = _PyObject_ComputedDictPointer(obj); @@ -1783,9 +1771,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { - if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj))) - { + if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && + _PyObject_ManagedDictPointer(obj)->dict == NULL + ) { /* Was unable to convert to dict */ PyErr_NoMemory(); } diff --git a/Objects/object_layout.md b/Objects/object_layout.md index 4f379bed8d77e2..352409425ee802 100644 --- a/Objects/object_layout.md +++ b/Objects/object_layout.md @@ -16,7 +16,45 @@ Since the introduction of the cycle GC, there has also been a pre-header. Before 3.11, this pre-header was two words in size. It should be considered opaque to all code except the cycle GC. -## 3.11 pre-header +### 3.13 + +In 3.13, the values array is embedded into the object, so there is no +need for a values pointer (it is just a fixed offset into the object). +So the pre-header is these two fields: + +* weakreflist +* dict_pointer + +If the object has no physical dictionary, then the ``dict_pointer`` +is set to `NULL`. + + +
+ 3.12 + +### 3.12 + +In 3.12, the pointer to the list of weak references is added to the +pre-header. In order to make space for it, the ``dict`` and ``values`` +pointers are combined into a single tagged pointer: + +* weakreflist +* dict_or_values + +If the object has no physical dictionary, then the ``dict_or_values`` +has its low bit set to one, and points to the values array. +If the object has a physical dictionary, then the ``dict_or_values`` +has its low bit set to zero, and points to the dictionary. + +The untagged form is chosen for the dictionary pointer, rather than +the values pointer, to enable the (legacy) C-API function +`_PyObject_GetDictPtr(PyObject *obj)` to work. +
+ +
+ 3.11 + +### 3.11 In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``. The reason for moving the ``__dict__`` to the pre-header is that it allows @@ -33,27 +71,49 @@ The values pointer refers to the ``PyDictValues`` array which holds the values of the objects's attributes. Should the dictionary be needed, then ``values`` is set to ``NULL`` and the ``dict`` field points to the dictionary. +
-## 3.12 pre-header +## Layout of a "normal" Python object -In 3.12, the pointer to the list of weak references is added to the -pre-header. In order to make space for it, the ``dict`` and ``values`` -pointers are combined into a single tagged pointer: +A "normal" Python object is one that doesn't inherit from a builtin +class, doesn't have slots. + +### 3.13 + +In 3.13 the values are embedded into the object, as follows: * weakreflist * dict_or_values +* GC 1 +* GC 2 +* ob_refcnt +* ob_type +* Inlined values: + * Flags + * values 0 + * values 1 + * ... + * Insertion order bytes -If the object has no physical dictionary, then the ``dict_or_values`` -has its low bit set to one, and points to the values array. -If the object has a physical dictionary, then the ``dict_or_values`` -has its low bit set to zero, and points to the dictionary. +This has all the advantages of the layout used in 3.12, plus: +* Access to values is even faster as there is one less load +* Fast access is mostly maintained when the `__dict__` is materialized -The untagged form is chosen for the dictionary pointer, rather than -the values pointer, to enable the (legacy) C-API function -`_PyObject_GetDictPtr(PyObject *obj)` to work. +![Layout of "normal" object in 3.13](./object_layout_313.png) + +For objects with opaque parts defined by a C extension, +the layout is much the same as for 3.12 +![Layout of "full" object in 3.13](./object_layout_full_313.png) -## Layout of a "normal" Python object in 3.12: + +
+ 3.12 + +### 3.12: + +In 3.12, the header and pre-header form the entire object for "normal" +Python objects: * weakreflist * dict_or_values @@ -62,9 +122,6 @@ the values pointer, to enable the (legacy) C-API function * ob_refcnt * ob_type -For a "normal" Python object, one that doesn't inherit from a builtin -class or have slots, the header and pre-header form the entire object. - ![Layout of "normal" object in 3.12](./object_layout_312.png) There are several advantages to this layout: @@ -79,4 +136,6 @@ The full layout object, with an opaque part defined by a C extension, and `__slots__` looks like this: ![Layout of "full" object in 3.12](./object_layout_full_312.png) +
+ diff --git a/Objects/object_layout_312.gv b/Objects/object_layout_312.gv index c0068d78568524..731a25332b3ace 100644 --- a/Objects/object_layout_312.gv +++ b/Objects/object_layout_312.gv @@ -20,6 +20,7 @@ digraph ideal { shape = none label = < + diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png index 396dab183b3e9b2f39edb49b033a612e66d3a09a..a63d095ea0b19ebebb86ccadb964278bcfe0648b 100644 GIT binary patch literal 33040 zcmeFZc{tW>+b;YQNkUX4LxxJC0g;(xOe$o~EF~pFW*I7RsSG7a6B#lk6ru=GiY79Y zDUpZ>naZ^Gv-^40de^(&cU#}LZQq~Y^|amH&&~B4&fz%rW8e4VjL_ew&C1NfOi>i; zZXFFnilS$vD7ru<{F8cG8oYqN8146HYfua1e@SKOu@ogp?bc8;^1b<^!_UmPX^ddZ|3G#vw-!68Ap0?!NB2zbNBIYkSdZW4Q`y2BDsqKYdu8TO!AB(zT z%I6{cfuXU0L1@h)HICE2e$UlM2P&`KsqePE)aG5)FSlmvV?*yg{WvL?_^`#|;5y+& z!sL_nW~|Uzd#48Z3^>^T^DlZVw7;JSIu7zV{U874T(@NlD^H}>@1q~?vJSVG{HpNh ziDVpVEmXI+Pu^Xkl~Q?i%fXAS@2*K!rS7w| z?Qm8Nm^@Bndh9)PK2F?pm&W^h=G&>A`}VQDy(v#mF)=frp$RG}l~!M?JD5nLU6hm* z6izlYH#e{R^l6Q_$vu6Q{*@dCZ>p+8Xf$0%7njJwV=cR~o9x?*-F0+y^cB}*JQ&E! z_n-Ek7&y0P`(Z&vMc%WFtYY3zF%-tG=(xDJxMfe=d{7{Yiw6I_ zsO71^zU2>yRa#ow_5J%&Q_~jRPd~Gav)OrhFT7KkVQFq|e%?GfHnxbm zsT%yWI%sTc%<;qKStUg-ZQXBgDbg-nP(O7_Y4hgI`uh6R*-DDa z%gh3;sBr$OEPPc@tY)#3@zAK6%rk;Vq$*pJ9Bgl<+xU`CvUtk*U zEX!WMm~xw$bh)YM!^SInqI&lCuhClZ+=`Z#)oC)+E&vsds2k{F8&q{Wl`n7knbt$*SmW2%*9GSb<2`78A zYiNv*XfUv`vAulzcI!#rNUgIXd#`u(M5}7;+!^v}vY=E~#`TilDW6hd~s_^COWz7m!gJpcQ&*OtwjZ4Vy~!{)R5GyXmA z!2^F5jujjRGBNMu$uGZlEuRa+#f3yf7{iwfyx3MgyNKg-TG|GT)WPGtCr_SS#l>Ym z(p70z;AD`tS4TuRdEkO6nag6lg@(3v&&Zc5T0}%;=7DtE6DOifaxA`_TW!5Fec!E^ z+qa`+JAJDT-4D@9p`!|%o0!RfDR}lLy*Pk8O5)dcjVpLLx&D^Wv1WWm8cT<{^61Ro zd4s*OSc4&Uran~mZyOMv(r7%OiWA_;Q~P!?Uom@*IBM{vvVl#%goIDSa5VDR$Zt-&@`QiiAjIk zqiu|Ad^DB7KS~XcX0VE|%r$InZ5M0lX=oHOkjJmR8Xq5C7BFT1=DN(s$6n&EU%&3^ z>kC&q%V<{-LYMdO;Q>VrgkbW3alO-{J=5J6wy|?^p1UFIntbHlZElrQa*Qn86fZAt zVbwZe;m51VMbCToz2!gt`6)kcDd#^fdR^8vs`Bg-hp#U$Ikgw(w|+sujtiBQTPK{w z6u;H#88&sEbCcMiBS+SWiZWsAslU6e+{6}10-5jg5L!RSO_e~NJy$nZVv%)y`xb^E z+!HC=8FJymh1NUdVoP!{266GUt|vdAR+akw8mU^YO19UH9WKuYkp`QEQdYcXio{EJoDG87=4LFj&K<;K{_V=&hln!=$9F z?EdZbnjf8Iyd96d4k$_s3nw#IB0guVDxV!&#<+a##rsxIo;5dfsm@QY+i{|u)|_)N zo1X`FJV#=x(Z!1wSuXM`a&0+SRNa(qnjgFn^rP+3a^$0bE~Z=gszyU&ik;)s@7{Rd z!JJ2ARp&na!>k_0a#}0J;O`xEpNnG-7HpiI#m<~LBjG>p#KNzz%H7?aTftK_b`i1y zCmAZHiut)2N&9LhGYgCEnV;p_*cFXzh)4bT-JJBoSrfwR*401parOM(uvm3&eC6M7 z#*uG2=giKYB9D2j`CcXBk%ZTPQK35K-jJ;GtT>O8CnbGHY-cgFAB$Xg|4a^Ld+Q)0 zXOVX-+=m2beQH#5cKpNskB?jhS|7RQ{_|s!bP9E?t=D3mg#7Az*F&DF?&_8%@>ChR zw&cfJYHD<0ENjkSR|qR8T;GjERDj^JegWMGw%{leTl~o1DKdn$xQ!^`cj{E0Xyly(|)R6graOFNd zVmCYc>m5RXbGSP0(BvM43o2AVR@9Ye~Ump69j_Wc5x?PJS#8 zF8}9Zt4s0HGIi;RLFcz0dE50mVY_qqF0x=p5Qv+&6A}`92Qt$~kXLN$lQsXj%dM}+ znxaB7q(aY~Q%kAmp0h0THy+B%xU8wA^~_dcWcY>H!SbN8SA(v|aL(bH8XC(hszaPp zH|)9Eg*e}P!e_8~%lJUkvZA6QZb^Fv7Dn|A8rpF}+qbV2I5k2yKN=leQCX=rMvO;~9yKdEzNXWE zaLdO}pECK`$eyk?j-U*UZY>{-VT_x?>$%m)PD=K<{q!BqX;+C40mhj`(i%8yUJ6w1yijFTmc<>;yrP8n0 z+sV!>I?-|diO)}AB_+P$;bE;I@*9y_^s=&#!d1-*HB3ynOzxST&dZaCI=U^TTV%=S zfTuMtUhu>Lo6`hAPEgQO8qKoUO>NT>K?K4p)zum${u5aR zCps)P1tUgXoOEz>Ogr9I>i5tvX!3`-ySFzN)sUn{9}yiLZxcLctfZvG5F+i|sIy|_ z%D6v$?-Jt?yqT$b=fVe`-wk)~-5WPP?)kehPonGK`@?}%-tt!xUb*7)kgBl9-92|Y0=KB+ia{ag#R=Yl*ZZ2#wrLtBxH?hOETv0jTDlkV>BD=r&0Fsi7i=>AUK9eZVrMq^4-l9o2w zG&M0{Tx)}H8Qb^azLlZ~W3;HOtZ93DJJ9{EA^ohhe1{sbzMHwZx%EZ6V?}mjt;W^h zEn~1Tu8r4Yl+&V(vrX2xWcODm;RT9|<NKIXTQ_g*tAHEUTJH^P# zBUA~WXk`f(V-Dh~DD;}<{uCXfnZ_1rU29AGhBi1p{ls(#)80@hjIb8Z^!}{JS zhnlD_H3Gp56ta2GuXn22W@gtO1kO&6_IH*kwiX($W)35!7qM)~ya~4=$YhuDbFmvv} zZ%!^QVJWEd$SA~#rQ z#}2N!nW-AA3qfJ0rmQ$aZ4I$4&0|j)hkpFXJJF%wczfTjU5l=6Hb29$POIzH#zc+i zwaO!HuI0uSr9izLE9tYih8McxZ`?RGKHa%HbVHy{Y2b`^#E_MxrEW(sl33*0ch_Z> z2^;=+M;^!2)HLM}%8#_k+^yQuWTY4q9dTm7GYaQPp=wSkcMTl zsjv6>h1DeNicZX9##8p)QdnO0uHo&}jLi*A3^GFqP>$sjZ&uaW5&Cbh6V3uFRJG6M zKA%obNs;va@i9DzO+!QG@1tU5!m-*!qkoo-UaPdNKrMTv2B>{YQFK6C`^*!q6v;n_ z3*->goH`yCjujX4#(4egw2H_FbaeXswCuqU$_yKqExOINOhehyyXM-^sRTUAvx z3M`}QwjgeLKeVh&nd(AaSQ$M3OW=^B;}_h~cV_63dw2EH*S8c~L_3qkIi7eAJ<}El z3`JbR+O})VFmUtmFtqn_bJJJ3Nf-P^6$N|+p$&2lKJws8YqjfpdrR)*HyaS5?-n3TEyvS ze8t)rd%^2UOHWTXJ9v-pq87*J0B0}DTV-42u~_H(%7 ziPNVC8%oQ{@)5xPfwK`~^1!oMrHCZ!BqcclriP9GLMfgLvC$D3x_fr-CdI)>@WNEE z@7Sx2{{ZqA-TJ~(l1}dhkGB@=9+Jr5@#S+B5E2riMMgd+<^8c^gSwwTr@VKPb`IZJ zSW$7BJ|9(XVU=L=^>Fow3|~CiHOxe-oj#}6c)!7B^9QL7Qx8x5_Bh`0m=n-Hdu+Gb zC4eN){`YFQeAxMm7mXAdgtIEIaqH*v>^M$Cb(zi2XlxgEiyLIn+Am+eT$4X3;*V=( z&SdHED|mJt3m0R}UBzzT*iDATgH$tzUa}Xsv~g`e{96BSZD4U4EJKFwrp=qH-``Qc zbo1uLTZ%r<+HblZK$#OwztFS7AkI#L8@b5$Z%H!sQ#(Z~@^5vO(cGmkoIJDHvX~uA zBmS`R?l9UCm60I|z{GevVXDSQT0M<$oLo*qjW8~=G^r_{T5yBpH> zWQlBw8~XO<#&p}~`7k6XdsIqHEG$NfsmyV)v1d^(P!x(r@RQ4zo62XM$A6^XzyBT2 z)Fbg%F+%Es!k|44V3w1at5E$IzejKp622yVbj7&8GX9{tz3Xij*WF*ALf|-fHgW_Ti*4R)q!_A8z5=A@PI?fi$MHf`$HG?^_t#+bb7TI@`hF7X; zXjrjz>sF?vOVy4azgIuc%*+fXV=>ZxaY>0<`iDCRC1#I3k0ykvo|T)L`*aA*H2CrN7=P{tI_^ef=7W+)D7E zu|mSjcki^pGH6BeoCBz`1rJQ9Q1{^i&%gqoK7ATP z^1$H6*iHIg`9596a4;)L^-gXG_<;SQy_$Pob$+hF;=I_cvwwfhl}!iU7aTo6D*lL# zw|9bh!u-tX)FewUfB3G!x*Q|{i0QDiXX$~2>k&uo!;Or`^S8CCa@0ErX-ut=b4z03 zlN-tZaqQCr$3;7v?^_)+Qt|kzLnnCW&K<2@a6MeAs;a%CSe3m1Hmd+lK9!tI2g0kv z)l!eO+$V_wwXDIwiO)~1sF%o8+t#dEbNTvpF%*iYf%ld!UoIPw1I$WFgkEA%tP^;)LVke*%^)Ui|9PI_>ovH)>A~wQaLc9{BqD(z3I&k%X;q<@G8m*1LA?(nzU) z364PEkJ**4c2njJC(C_DlQFZppv(^e$X~v7YYUjOIBegeAj8be&DTtruBmzYvgcH= zNU}xW)|K>9qjC$Qq~J+QOB)~VIJoIa;7pOyX!Xjm#sm8!_NB*SM#S2{rwG~v{Z1R3 zjx)91v?E_<-##vJaq-c~Nt4|aw0aIPEF7tndY9uu& zsUkHxIrs-qi`b;!)s1fhW}c`@j*N`NVDPVx|LpBe)3tX)1ocB5oQ5ZE$al~l9UotH z?AS3nK`Irsb3=J>@b9j(%fF8G)B<|lGd!%mVk)tTIBiCt1+5Y&LFX#Q^nv^Fdd@Tk!z;>qA*3~td@a-s8 zYiJznK;kp=$+=N2xi;E{hAT)8Ub<2w2^c-A+Vs}+^fb4!w~x;f3YC`CCh1FYPr71f8>5SK4vmNRLvxkRSwJvg>UaRa6&4~{i^|^R%>6zT}c^(8A zkWth;&#_x^agZn6+(h#izzHPVRfW9-VFF~H=lARIdJz%f?b|u+#@nNazJGa$#4Gw0% z4o=ecS@>;t^ypGb>!fY12b)r@ylK1qY~J~9&?;d7rmwsaU?}~ktBOs{qck;tK1^+K z_4eLDYQ3r30ZlmvS1(<avZHLN(pXYlGnh>JT;l)Lw4AxG}*76=XD$T!o8 zV+V_lpQZ_}U(bN?+-hoRq#W1Fm+|uC@O)r=cw8JOAgmh*a)i$t#}}Z`P_q+z{WUZD z11+EezJ1H)Gu*DWZ(r1_N$|$8f8yMKM9|hQ5n)VK0H6ewBaBKE63Px_bnrVA1<^*^ z&Q6FZeh6MrC)R4YsPEbZnJeVlHSXA;zBf0*k$dhWB@vwt!pZ3yH+Yc~nO3h}-K)Nj zsCfCPGw+~q!-^o>E^_{vvu6Q-EbsT3HOmQ9*Vl6(iBQzCWy_{#X2ivB{Ae$EXenth zVbpvwDfw+lz$dV7K$EpSpeK0}gZ>=NbUAl#7!%HSI=KAbLjLXJ%NkqzZ$9>Z`hLTRBTk^YoQl z+mEn;3_Rar9l#1wO$+Rql#~?NYy|$A6}t&xLhT?uo#=Qhv|+37hpXO02YAm zR6R-X^V|q9yMPN2tYhNiO~3V-Hp|u7gQAcNnA}WM#GV(duM<@TgzYa&8@w^TdNN7) zU5T(uTahLTd?cwbjB$61O|T}g_8u3YD?TIw!Ofdl6up1!#6rc=8EM$FO$ZBaPyANn zuqmS}=R5n56PIt?xDiyW-RGx)FJHZ?#2z_=-Nva{njsa1W$Z92cUSJfL~?W1>f5(( zH)I&BljLU6>eUp#2u%C*qxbjk-=)Ao9&JdCFf%iQp!v6ULm<33kaqWOZLxb#bw`I4 z6wd0VCSy{KK^dtPQ|RpM9Ghsd5s$tQ1oFQZC{Cnz?~NVUuiqVwugb}~c3K_}#;W1u zxb*R(8EEgDU^y+p9+El>N|+Gb zUn9tloN=$8`>_)>Qcv~mG=E#tZ7=%Wm3Q^(jX*10px4}eeByz#B6LRkhA^%73b3YQ zf$T>|M%b~s+}+$_NPndLVq@1J42Er|Ng__8*mXZp_eC zoYX891Mi>wgy)hyqsf7s9mpY61@NA6RA^QfaJZnfbiqfzHH8UUDQZSWtDx81^A*ehHeGSFTw-x^sGvslJ217nX z^2UcrkENdjMCsapiV-N~&fy1!wpbSH%iq3ft7E*|SXH+C&w%&_6>!mI)~<#d3A^=Y z;w)m~j~_pXa0#G!t?>elCNWUg(lX_)v{+cUNm4RNTMAVZjcImvR(`18A66oh zh-sK8gH=@5c%Nk~UZ^f4WoY&G4S>74Moet1gAvF(ch4UmISAHw`?Bc1Wyu*N%Ot2N zAD{T-8gu|e?$Qtxh7b^N^UF$(%<~a+#ylLSD{Vt6ZGHPy5RMceMx8d`h#Mvx3`mY9 zwzFa}V5Kq)Qo8#4X}~pn69Y}^w-mhcP&G=sw64M))v~s}-DY&s(^Cm` z7KVi@0QEc!|iLJe(%V9glZz?RRDs%WGl9P)TRS>lOUZ#Hs6IbTaT$Y1po%& z*jPlN6V3}sNl9JRZ{HlBZ2Vj{ckqA1{8sh^zx;<{MLPEB_U+pvTalk$-&W?OkP~c; zRNT6DrYVVuG0)G>yPZ6F78)otg?uQ2Qmlp7l>>?z`pVR^pUw;`rj-IKi3}MYI3Nid zhcqJR)vH(C+}+inirPCkL=N;KxGCqfM`11Ea5tXc$b0UOmmjE;=ZMnS^P7-+J%-!m32g;7 ztqwKFmB(1UTuYWL`GoD!{xt9)VDfq;B}+;N<vF*JsT?vh^9tA`3F9m+C&1NBZ@wms)P%FqSIe8wn1_9g}KDs~~+dFYbJ$z(t>aznpS3tXr zA!6?5&h;Js6v0G+U@+1N2A#hqe1$dtik&HTdYTr6&gUS`6BJ-Mv-JXe1Us{vu5VsF2rC4PQeBOi2OtA5P^cAo ziK6^5-@*OIN6FqQ7cA||n++x^Pky+D&`uF5S)}cJ?fC(Aj;6y9lpQaFEykk6!do5C zlY}X`di3H6i1nyvZCzYg6&0W8Mm#K-zi@v=YHI4W;|~i9w@l3w;nx<09r3|f`?av(ET zKX9zW_m&XfL#^^A1)t%I5V45yWJ3`ovggQ#V$$A)tZS9b*2G%#Ay?&?EaOna29y9h zfjtY~o&(aB@!^h{)}Rxe+2#)qQ%UQ2r^K%7Ti@O$3te8!S~5v0A87y9Eq-Xu4jw&% z)XSGIMNg{2l>uj61b@fj^!|fSyN6xh-`&frcU#Ha@3@15!?<0~_w>XU0OuBW z=aFfS8r{ac6T&S0Ljp^xRi%)A_;IhM*UXycAHn;^I8$h7q%^TL08~23* zab6yQIC}M1FtY7-bE)kB*-+@7?Cyw(VPA!hqN@WGeW%8ZOH$F#5VwC{ZGZM`5fV*p z*4`NOeTH zcJuL(UE`1Z%L@o16|gWj^#zq4U?cI-fE1tZjkl42YtbXnLE#@bZaen%)yI;P8!^3? zu3e)6t@FQn^-3#34aw;>0=dXNO;3I}I}3@pqLl*s1>J99~4`Jh%=ZI+I;pUtdp@n%cr#useV4dV%8{ z9v&WU;QCgmroM&d1mykvSZ-NZ#}l7*;^ND|RG8)4FYXNZsSNV?z0EtU6nO2+5ISyb{xv;Tw@rKw@3$Kvi4b0>Ase$z_rtCujDgac!J>$T5^({^6_(OE zZ0Xu@zTuyrVGMz?vtbDvpZ);Mb_E}MMFn0UQY{W49--$a_QL^y-5voC0;y)eqg~{a zHv~?&GUp!jedO0%{ytWdo-7BCqT|IMBAjx6%Fe>w3_&4?KExQCf+-OBdN@ z`uOMv5qYnHM)g8XPV@WY)`h+8n_Ws8gwyvYUPTzEq2_>}<9mHdIG@U?;>MnuUfqZc zu99qyO+A!hmdyF9Qb-diwU_jb3+<{HQJ-J+oAqJ<GkZMbe`+vn z{}e+g+OzZK5ftl@k_qDuk^vSl4Lv=*a})bL%OSyt^?4ZIoc)7I+cXO3+&}Theh_ZG zWk~Q6vfKR5TWM&78rWbV#c}T7Fo=6r9b)7NT?nR*1M>Afbxbo*_@@My}hqaS28rV$P|r zeXD?p08NAi1u6I(is7O;7Zjs_5DF2AiIg|RZgi&iEs3f5ea=ByXjG4$O{#n5!lguG z0)~LVl4gMwsKJ1csnr|5&E^G+PBfHLz*O_1h0*`34$FS+7D3BebzfH_>pFnew|9xr zM27)KqFaKYj*eaqa-2Az!LoC3cMQ@PoZuNDJ(2BFt@b4*)Z9e z+6C?VFfMR7!9M@t!z#Rr_@Sk5lh;>#`t-c553iU|ytu8;Sb3zWkru0n zOz+y)ucZ}^$ndZpes`Od5)~Dd5EDR=Ww`dgR>7Lz+*UsLooH*izkNOjreA5vJW!Kr z_1_6<24(-P1{QZcbR|cVmL7|a43aQKZQPjd@u)0lP9fy{n){IOSUA*BnP&E)Xe5qv zFy<&I+(x?8px_r&{{gMf&Oyxg6k+@Y1O!M$0cK4T+#U=QY%m`KRokJNVfjTFY$ggm z#A6R~oH(zlF3kQCidxjffC*yfAn+J@67caEtk)C;(8JDwrA7z2wCe8O8{lE-f?mc5 zk);b`8Nmp4SOM3h9GUUc$%??YFy#r`d+eLUg@hLoFVqga55a3dsw1!%ty{N_9u5*^ zu#yDVBJCMv&V0M!+I@Dp3UO@HV0k0E6ymSCzCJrK4ye?Afh}MLk-6zg!K~9IAavfq zsx{Cx4Yd*OO?*YPqSr=(ny7%mn8YcNYi@9zAS1_R&W(Tn;0}8e1ak(^Ft7)P!$E^f zgd_FZbyr6$x7?B@ty%NSeeKGu>!`6HUlXz-Wem(P7>fj!O1wMo-TjGC8*lqALKdtf zB^Jm_J5r^C|36nFi^^*jstW1Mfa!NxnRPU34qKoyb`HI35^9+NT@`es;=-oK_Ju#; zIdXSbZ?EwecMgswZ!+te>NdK6Z25V4m;G(0(JN|#c+prCS6Ne2)1Uw3iDJltRhe%L z5C=W=XJYV((FsS##gH64^YI?629YqSKE&vl!IML0@Or<*QZVPaPorsVqq_VDF zxnI<@ABG@i3btAAb1#g?EhM;Epw37ntTY%8azoKhRRCj+BreP@L5VF6CXR?ton-;y z5j((x@{y<_>F@#-qFB|ET2jNJKgIOL?)a}ck2=qoWE+bNWq~+h{$KUa-lEm4LLkY- z3lqWN(}%oz(>c85*RNjzjc%EYa21`y1gZS#Uk6BP2Wv6Zk&CzZz(9 z=>8fEut8F5mA>ZEOtPON?(eO=@a_33QhfbSsU^2v=f`|v@<}TQ^}s_P%Y#*c?(#u_ z(U?HT>xJpU`JDoLch=F)Qh6X15|+Wmmsk5g&r#|I0zf2My>b8TYM8%Gjz5{6_PuA8 zzX)}b$j-og!m`5E4YIiuoaO5Xhac4mkQPm${}F@%PaC?h(^7wHc;TMWSeLL{u5i) zjYd|u3SeS~RZ*`9%XoPtkBs54$#8#v#`;N=bt6+#rY6*rg+id4=&%U7OlQbU*A%?O z$ksDR(fR^lOnlOqg;AI$d%!+uBS4@VL$7%_QPP1CLqU8e__`q2W1pX$xb+H?TgkusTz zD9Q*aODvb(KRYAA5ui`T_W5~E-=AeudFUxi@neT8j8rwmzXr3e?fmR-gzWzMo-b@H z92}hc|0gTh<Po%B;>h;DSY==Y?&bH;0y#IK{gIpMh zmo8hj5<^O^f${j>lpPOgH_5VQ5OJX)eZN5dqerXDthb>?i^1*Uv!k`qAFu&>z~Y@q zV~uQ8FaOV~XH%{K2*@?^9;uj7(LMz3N1M&^lB!$I(sW?c1NO1Ngc1`1nD6!IWMD_O zNt-9e)<6Hf4Ac!P9;vBzgM)hh;)T}Dn>VXlT1+ven;6fkWj;FmhKEiNHvDaqe(-6O zeQeoCnU{DtM%28+I)jLAbJt-`E@Kq6NZ|{sNFDhlJ_jOA3$Bnl^goIz?I+cp70_S? zWX%FO%hv#ROM>lZ9&BlE=OYa+!`jI^o~o`i*ihT2Pbv>#VdB3M&XS*> zpDI^RsTcKG6wD1|RXdjXd`l3TWe)sph;()Zy`)ul>)Y3_N}e|Ve`)joytH{A_~YM| zN}mzXs<6K&GO#ha$-tjt{|*In7=ari;`glJy=j?UJs)y z404K!raHk>2h03rJQb|c#`j|P7NX_N@C~RnW*F(Ap~qRg1M`P0vAl_t0?~8BrJzjW zTL1b#g|k@D6O<+I+0RZT!40*%V-mqT;Hw-=8_9a?%}&zbB=_a_n&w(-f`f!D70t#M z2@RahqB(&#V5VxT9c*lD()RiLE8NvfID-iI492$oN}~*@G0*n*_iG2O#baqiqh%>m z1R692SA@}U5g6(v`jS;%kf4Nw(S|Hmh-O$8KyH}(cR?3rTdPDv9dy-96-1Ws86YA$ z3hAw@r{^qjlOT7XRkUifr^XIS4*=Lbi^73RiOVo<^;Kj%{ob=D{spfN0{AH)}8;9!h*N0T4E9GhswMkWzz0PR;vFbgS)f?PQ?{Q(;g7 zrv<=K3|!8D%|<&@$)NPskc*}5Sxr@-?5*o4!uqE|9v*pT;-r@b@ma$EI5DO;T69Z; z_XUglzgbd(S!GECGo&Xo4njByoz%z{9=Uv2UvltMJ7ABcH3y(Q20!9hy?RRYj*1jW zf8fO~aAZiwJ3d)UhoV*-au3(S^Du;9k}^<4<3%fe|LT(ro}XPY$oXhC9XOsiI*4`w z>$Q3Q5Ws0ZoDPV%+$bNi^NIzU{~NfoO=cHJAfgEmK>6VUnmmt}b$}mT&TC*5B7Y>* zGSZFa3nG20=-upFtzpa<;!{g6V6bV_}?i{@d0$? zG(6nCO&^t;kXZgxCb$Ei%ijh= z4Wr(S%iOeoe8N@hrqWsl>jCDNY3H4j!RqYNkU$p^_*S6B@r6R4K6eA-g@p){EBIJb`})r}3-jS)&fg53?suIL zWPXrarSQ7q_grQ4EBBbahj!Y$t8rK>hlo2oBQ+H@^+vhR5@qlm1iLV{95vrk`0eY@ z+Q;;9!2YfI$jld!B^dKvB%+o>g2Tei0d0K8KMQ1#mnF)si3p zi6Frf`Qp9M8Vx_ck~{EL3ky4QWbI`^o8KM&)bG&1`}>_RY3!g0v6yx)%+I;Q9#J*{ zrQ+|Q0Bvo&ckkXs=XF({U6mX9MqO^fwx-!uq#>+^4#>jZh=>So)gUE!dGF|p5+Ur? z6Bc7hE>6xbkPccW^z=Mk&!CyF8=6QM=0tDrUR2LCK-?Z!(5Ehb2e2fbyj9HSiZ+Xj z56%aDVAKZz4?WJd|NR}btUQCk9~48x;otgMe@-z@{MHMen=qx`vndUO1uuGgt0r4d z9HtYJSq&Y?+Kz?7qwSbez`r%w!figH7!2_r5(h|$9 zXlLKF@1~p(obZ~^v-=w}VtXI%5NzZQ#4E7nK%O+hSxCdVz>IsPV1uk3H6hC+<2W7&QH5 z$*nOs5QJi1X@D^(Yy&CaQ#e$(zG(wv(}tyh<12rb{)#I09=h(`@hyhCeQ>)|{}KNw zcN4EtFwf?$yud3>B8(Ht_j#{#OTQTHyh4r^;86Cgbsh>AjIV=IcI4IrppgO`tbmSA zZE%4wSmvYWimHI^)pMZskQ7n1O;eCXXo1AzkkDbbT%$7&e{s}4F$uX#Vz`W`T6yJuFlTPl-KQUqclSy zRn_v=qTyAmSEu`MY>Ka&7$6ch`~}$`if)|!^ZMBV8HwL$DI*R-*gMBBSKc-0IO3+7 zc+_jCeb6`OSXmff%alj?-@Q;jm*r)W47Y^ylI+8W4pA9;?1!XCyRN>4#d$O>ls!g? zM~N+DwS8Q_Vs~M$R00bcuq7lVPg~@Ku_w2hB4!L0+yd6@co_L@+M>+g>(*qp^GFtG z`A)w+ej+@Ao;zDu=+&VTBGPD}Hm6t|4?D#{i+NWsfkj1xR00}kk5StFI2-`)Hm zFYkQ``!6W<@iswADQ;zdX=$Flh1*2~r@w@wEpTF;eB_ZPo0N|B_K|_?+-?{%27x(i}65x8Mx zLUEopPGW$1v>SJNW5HSnt~X_7CFyLF^BYUG{tYidXvUA4 z)L%T3_Mt^(Fx1lE8D&N!R)SdyZ9Ml^9zd_1Z*QDwXncHpjBXlH?$KIt0R{o5L8C`Q z7C|y^I&VE(VUA`$oTfm7JLx=pvKB2e)%P?eezOdWC7Af!q+`c8Ha3^c_r$*uROt#TdU_4O(j(g2g#hM*SbsqiIo5TnLG++|ZU5Dx#+DJ~Joc=UN5F6+**v z1=)bhL?6faMP53qzY_DenjEbG%QfkxH_R+}I|ziwMdhJ5S;D!t&<(;#B@*C`-YWE2 z(=yZ=qqS78YrjWW8o&DyebqReB;KOtAD3r+Rc>oFp`T)O*2@LYD*+R;MrfRQ)ajp{ z=|HfvaI%P@i)k`38iTX4I8cL=7Vu$wcK`3+8r9@@CVc+JM+EW^oIwSLutcGAr>&yQ6;R zurny&zsl`eDL$MWK|cRU$OSM|K7)lw7oo8Te?V=ilj9^b8aD%^X}^WBfKiA6TbG5O znIzqZoS<1`q5okg z6J3eriGFy|d0Vb!6I`%(TBPrQPm9oyI9Gxk5rIOZ5_~Xe4@)!5%<+L73Cp1^&LjfR z2!}5hhXSyn@VqrSVDTUB0dENv0;KzWOWVUE%MpE!Sq4L&mOC|_XYB_%x6xS!1aW~Q z((H3AWSHexZYp@`wr%MLCWG>o?NfGD6csx;mk7=Qa_S7+H-zv-=V=%@59gnmk~R|K z9Q*YPPIpFe@odII^e{w|eoYApc35m%PhUZUhXR#UTx@o53fry*oOyU?CN(iVijpT3hoT@e5t0Zyum>j@JtL=80ZS5YNI$zNjP1sN z5M(ZlzcyOqzk0}T-m-t9$qg9^TwhOeX+R0W>#kA z6(Hml9AWk2q%=^~z|V!5;@rg~M3_#BhOlMSW@2uh0C}MtEsv|IYP4m0+Du%30&4z3 zR&I21vhXFeVWFgc4k%HL>X`HsRh!xbZh$vX5Eh$tk3GK&L(*AHfguqhtyUznx~x%?j{iP0e2J?i?5}JbGY9}A?p^hZx`+_+IRPbl_Kc9PLsrQpAT-Nmz+P~(&K#WWUnELz VMgU89l*m^wF@WUjRBn;ku|o^no_ILnQsw)Q zzCYY@YYo8TNcqBCZmSoX?iQbdV|HX>g7iKUqa>4%gl$C#V^hFKY>i5s6bG4lF@;+0 z3liE3#3_V7I+#z%fkxZP##expWZ@f183HN=336pJ(9JE+`WDzSg~Ewv%eFzQXSaIn zsfH8ckd}o)-)F{>t|*-IzzN`uDL;dbF|@uoK(r!K(k^v%Qnd{!Ol0oS(D1VDdL4TG z&ec;jJDG&woYXjUC>8i4hon-Ne2Gem9`?OVqx&C)DcZnIxMCO6rG$h{hhbwu4i3e3 zMw!x`aO5@zyr1uzS=rdQb}_lb6inQe^-z+@B81yj6Wk|RrD;QFj1v8~>{kED-MAi{ z!0c4{0w)6BF5ycwxIOZr4{^;XI)6v2Y0Al{#rRW7J`?V?r$BmW4N_t%13KWMyf(Gr${BiZ@QUudQ6qd1bmrh1WEA&*D zm~y8~Kfe$RtpF2~2R{{`k}o^rM>X1!M`3}(lSu!W%<#tdfOn(|d@5CKZ3kz9fGtJE#GWn8PpP8sHM`X-l${j!^Yb&V8Af6d`M<9}hl{gj zVVTK*E@L#ExEC@LxW)WI_4_9cwzK?=_W)w$+`6dApP!>4hmyA^hchKcjD##^U=W0S zN&Lhp8S~*&MCMLH%SlXKFPvxG5H$1PenVyLA}gHqV2?dE;l;?vQABQX{vJWuI9TBf zPRK|qEhXu`_0O`^TSGxA5OXXxnlO6Y(0ieT3lVoec-ACxWFf3T`B)NEhACM_{59M$ zYojEST9sZ(G8P(SHxa%c2XHcic_+@_eu#dtHGo9pF<-dpg`Le)4L6Y^4lPk(-M@dI zoMOkd)zU0|;|N-H>dT-HC36v!jwat4oab3#m8X2UC)h7vbDIA+BxK7g1Re3oNJ* zDNi$?t1aiQ1FOmVjjadkz(T%&%5S?TlRr5&1$-51equ?%8kF@UN1|<^1J3fF#(ySV z7ymwN1+Js2#>U3?c9>6@BUniNUnYp<9hC+G+jA_a`lo>b>fr>=S-L+q6!!PsdrN^6 znCLY7BA^m8v>V(}Z`cl(rdB`&zs;7Ph0x8_1IguoGoZZzd#OUfn}|~y<|HyoexRFnn?G{1$W^_ccBi@*7^nh8COG#J^5vrxH%9? zt||I#ZD?p1Xnz49N2*WZ4I6stwD92LU4+1&D0F8F2VcTuQ0U4(ew53Pag63)Zd!D6&Guod?I}5_mGB|{B z2qy?SINoIXd+836i2N?;b!I1maQ%xEJiRd~putpFv@5sQgvz`dqv z-pfKpg`A#5<``iJr|eWSoTkyPFD4H?wF$6aL`0;pdHaqXVfp!zVq#*pc6M~gM=0Bq z?q%FtnoN$oH6L1t#nz1+o-hqYnUCWajEszS-q9y*mjHsAkZvCP`~=WEk6a~ufAd?) z09*Y#G!UG|f?bxOzXXa;XHwPmt89wXg?&Ou zT6z@(MwqXZ0frs>^9cApN2Wed;ncLm`V|p(s+_KRoapW9xc_sP5mTx6v`q1m64{KK zC}Ne^K%qb0ugC1X@sf`1W|^lyAsP6)`PQHsTgyGuJZY9tIQFzIg4SQz!;g{-AM&K+ z^RI#r9elVUjdb$;*MEm-FbT2nGr%o_ql)N(sl(XLvaAOb6*nC#VMRfh@jGO@urNKD zPCZLK_CNL+C^1Ef=08oquqNrrji^Dc&7y-rtPxvcb^C{XzkWtUPO!H!sNp+Rg2D~m z2*>}`vytL-{Qd8lzVgwIU!UnHy+a~qmmWW>;OHp-)%MO|L3`#BM?9TZCJ!aCFn#JJ zr^j)dQvQ}G>$6f;(0JDJUfgu-F1M7Wa57hf4<2Eyl$Ey6^Q@c3BTJhSN41PopFKEH zH_{urjg{TJYQ5IR?zZorpUJ*nUDti}OU`!J%A&X}RM$*!V05vA`?ZM=n@{-;s=|0O*Uw30(chAh))_JlMR*M+&FJ5?F+ z?QJD!g$}sjT-Q7KKU(|BpsK>J-$QpOARr|mNGKs92-2mZgtRmW(k&(3D5!`CNP|cV zNGTy5A`+4wx=T{xNZz&ey&vwp|Mz~lb3ZuFz|3KvXYc(yYyE1u*eA-fVO_jO%K$WJ zh#|r*>ru@QPGaZs#NSFFqC6vvKLNXq&Ge7iAEm!1#m{(LCZ~Bz3{Q&r+)Rx3F5h}} zM1{tX%_P67A$i{2d|%_&8X-6=Obx9Pv@!9o2$VxPl6{l=?CnI{=*RKezet9yJ*&b& zIUIT6#O%eOQ0(UHKFNQa>&%9J4z1lxc#zOkmUIcf_9klWu(J62#IT`pab-~*%X;es z#XdI52z_$)rrb^EsaYJ+nzhR=v97Gbv6=S!l}k4&E*&)=n?5}1nQS30BQEuC^)o=b z68*9MBT#ld*hAHk)XSG}40{}fs`uT~&sq~Xqwd-UuS+i8w>uq8`I}k-6#}h0yZzJj zGm*(jH!!>0Xj3zqnb+*1JK8?_tE)aNfXy+N` zc^!>B1s=$eHxRVu<8kTEwz9xMaNj1`@6fw^!3FiaoPLt(;=Yek(0bKWKSjy}SAx|E z>CU3yiP3(pP3V7xkqO`3YAm6ka@TLZva7IbYH2MZ&sZh2qD28}mG{a@5Vu#|a<(Qd3M@k11_D9x+Gc+!XCRO(Um$8m%cc$ZZ zXJhP!1kq{Z$;+=6PFt>x0~H#q7dHEk(#}7wkBoQ=(SlHpjLgf$W1;+Tj@&x!4eWOK zLTp`37x7R5rx*O|EQG4M<(CS34@-`< zFv9qmJ~D?&UEK%^rAu9R>q!*~#rLZk6&ZtzCFeF<;{Gx0SsKZ!ZVpy{j^ajV$}<($ z{;rvzZEeSy`)dm%W=OGmmRmJyHeGkU^Ds#zEnWxx+0mdp!(EectaS5&^*W@7ukO?83dOOs^Tr zllxTD`>TwzgLBHLVA*xKypPoRvRXx78;`rNHAgG+t10!w^+Y@wpj}{{m7}~Wi95gI zVj$(m6fPwZ<@nw&_)@$?WC?Av37b&h=ja=n3Ot3k^sv}t(T?e^d+co9Bc;%7sFP?9 z?k=u>5K;K4&=K)1*Hv80|Mm+B%7Q98uO$~d{qlQogfBEevMdW@)Du;ofXwN6Omw#Q}{N0r1yxXA#-~Cq2+pvvC zeiRzY{bE%@E?c*_&|D>J-|ZvDSxB@ph`qSIMh?*0T>Y;JlW5)-t@eGzq2HqKs7VQL zS(1M^*%juYx^duWJAJ+7*R#NUwa_liT_2HIajyP2y5mMG`}oW-^g*ZgyOwv-SwGSR z$|%D#$z=$!(!NNd#a{4}6?c^UZ!LfyBe^|OiF3E2futvWcISQS#dEVQRhLRO>JKTm z974KNLJr3r*)kVY*g`y|DPCNH$aJJlceZd&I9c1luZp2hi(FH#WMAJGeO>Tv8r?`E zw@A?%!~c8=-DismU{>9iZg`1GS%h?_%@LEkdO~>8^JRE@{^cz3?#8l4cbMn)T*_C+ z$(zrS_tSSIJ4@gsEjY+-|9Pl8qGzHucVWltgKC}GmnZu7UAb@8c$rOo!|D5CyxSv~ zd-k)`*(9Thkf=|jny+=N9`US**0QG>o?dn^u8y9aWK=qe^qk|IZ))knRhdv(63iy3 zmKrzAQC4$y5Jdhyt%r`JYU5ZN<9gsa-R3q56LPR*S0%Wo|48Z=dnpJG!He{v(H>3( zHem#c#j%NM33vc4n*2al?$5ZtCw+uy+ds@!)DQw3gkRk{?0B$Bnj%FFi(nsQ1qaL* z1a(%=(v2(Y>}92dg2Y7U?=k(~Y6zbO`ZF;yU>dt{G$*03=9gZR1_5O z0D&s7SLKNV<}(@$Yild{SQ!KaIz;B!722CP+ERdl0SvgSb0q;a&&7i6UKSU5HR3BPcAn3U@y*2VHZTE97A6?Sps4DY-gHco1#G3)!>*?_4y}M z>M!L#T90FzeCu}t~DK&Y@gKF{lyPdq(Xw+!(QK(GWba_2&R$SP@b8~a0^FHpR zZ9nZqwznTa_o->A*~O(8@^NW2@#2c%2d}x5)eO@8m71G=V3R*5ho(9xHkgW$iH8aw z_*R5Ufch2se8j~S8mdqyl@m_$H+&>2g@AsjmvgdOl(YT8tM+i z2ksK*C6=Qt-jEM^2v!@ndbM(O^2w^JoO$~3nOcHjL{p(G^}}-)gf3BuXgFkI?h6wC zAbA-dLvSxqvF1ZXkuY%actG=f`dr-z*9|1e=H6Zyh_wL6Gcnt?GPo*r{>g2q;|Fko zP5}>^g*`SUuza6FM8|G@3hdgsi;6Rxw==XeU34H&exy8dFq+vPpt7`zrmZZpGhH(y z6Bntmbj_GNt2?4S!R@@`UZ`;#)P)VZc8Bs;IX@f8?f&f;4{-|auiT=TVVSv$^Hv-3 z3AgS|^|j$l7)@9$4w8dS4K7L%w4M;rf`l!Sz z^&$mJG}m!ma|K7Ss1U4o{l@6vI<^!QTL*zC(ul5wP@A`DHO2#fG;Zny9)|} zB&QeW3I5&P^WaRt0dNC}^El(nc^({3aw^*7C}{`wa3~}geHr`aKl9NInbIYXiZ>6f zrTz5CwfI<45uosh#M5oBSO2}bHB=Ij<+j$hYF>ry@GfJlPgQk~TWzcc&eJh4WJT*R zxUHhe#)E1ypkW5o*0wCo3y3a&U;0ktA!!5bgBQfa&gb+bw=E0h#3ZGVKsEj8;8Rmk z8H21bQK&6}j_9FSb6d8T@ls!~DO3ee>=_&zKJNTBINx!%L;hgU?VXW|!mrP|zfL`+ zwts8`rZBa<7P-zk-go5_1M`z#f?!g=RzUwxN72X53pyA7qb-6H%q#`t@hlf#avh2l z7oLXcb8}(qJs#mY6#P*BeC&}L?9{a(r8TDggsHx;6nVkk%kP2Ys~!&aWuLQrDK#N= z-}c$~B`DfB+m@&Yh^B*^kWBxbwNYBw%kZBe5%(b5xC+}+spF(1GEE{JQlWl)znGBl zwxO)x(#=D!8ROrlc*V>{`^lNHxvX{ub|yjt_*il{F&ajHoBQ^ z2y7feWj!|(cbdTvp#nT896fjUf;{*g;Y&`qdf&@@|0?Yb@|SLjCI)pG{ zCWmht@Rjxj7Li%YrhB^Fg;OFO6en^=)5DV#3)dII$YXL96SopSmG7VZ!1yT$6T3wz zo368AlyKAeD!_017hZw$R9-&p8n|*>;#y4)s`|;8h=ogU9o_;x5dB}!kBcH2BsO3p zA)55tzrvSXgK8goPI#UvBp?IJifZgVI%nr>1uj%Q1~+{0Xg!QtvL)g#bmAc?Y}Oh% zNE~1E?qxjTzeKe9Gk4H;6=JOL_SVJ;gp`Hcyfdj5E+^p>ry3<+ea%d4Ib^xS*hinj zE+(gy+w^B`(b@7Febjjg;!5D_9S&R(kPD#L4O{WQCu?pi2s_#V_X(kh_m8MoJta$% zluX#(rN$*C3(#!a!rs|@rPsMR@~$2E_Y!uIoT}tDcrniG-lj%UGKY6nJ>*CLN6NH;G3fY!sX1LQ{!nr(JiS{cf%XOkM6ZQqXIrG8rxAeV2QlNB2(23h zl%h9FzE=Kx?4|=vDPEddns>4IR5lirQp65SPmMKQ-D& z3$sC#Q-&V>g>}9Ufwr+XONXLRuLj>-JF4;96XP(YM-+15S|Zq#G}xJ$(^S%5rcdxQ z!xSV+_LiqF3cgC;-;5z>xy1Qx)6A~DHRkHfR8PysIc!nKy1IK zfyb;B@n#c_zu&ep1s%MEJH0pQST(YjHy_J8m$12vTfQ;OoYr&5SoO+bI{fr-I<-G( z`>z9Af)cd85Yr0x92c2b0HuXkanhSNVE{MEuJg73N-5JfxGz1BjQfJDE$WI;iER&G zSHg|i)LjJ9%1z|yzW5~%_4NS<_NzEm$B zIWVI6nn!thYCeEj!#jO4%I_uJ?UnI`6gXx}%)gQwoUC1qii(~B7@=?D#Zb*H-Y%Y& z6PiP3>mH7|ocZQ1H0gvyo#5Oqs}sq<+F=rN{PUF4iAX?#a(gu!Bs!SbXi`GZ&4bnT&`Z6U`W4&4#=jPB~@Nyg`!!%gZLc@4ZO#a7>fnJJn8y2wzs zcKV-y5td~FTabt$iQOOqBmiF)GJys1HRu744U7C*$1lgJnivJh$<37qtYkn?pAYSmVWa5x~)Yrd9)HX3clc`^FsTs?WsHS-79ye0hq@zVh-5z9o1o& zopF6;D>1oxkmBkw>Ugrt>0V#OFzi84Aq{-rx zMc5Cf7nYNB#VxKRqi;;d$(0_{j7 z@I+=1`6WidHVT0ush%@L^6M|;lcuiUW7?Atp2HN0#jn>o7599`gdH+Ck+?OFNw^k7 z=e^(i_TSJcWWUE~&Gwn!p{TpQeshv4i)stjcnaO9wdhmaaRbaD!-Q$x3+ z?s` zOJcM-OOz}$z#W$@pdORO*iJ7Almx4cyZfR|duwH2xdIGWmFnq_DQ|MpDGf6!Sj$)k zqYHn)MZvl^4Kz`&9abp82aJh5ApsOaY;ED<{M6U|)R0pXe?Ub)VLqFzpz|mxG1e!l zrvtwK8Sy&P>1L62v<_g{<~)XX<73H9^B`Jl9~M{(Nc~VL~hW+ z0cQJc#_vUNK^m`5vk!odO`Uzjs+WDSpHDoG{-{bp1=&7`BfSB5L1Z8R2=dJ#t_Kmj zp-{H%ymfA*a}IoVrwToTD*iOOz6?;hhHOk+gaOK-Rc0R+KR zR8t-X*x@+^{(RvD(CJSMONu-~3{TFV?_^`65IHzRJv1O}E-+E7S0u>lEO%_4Qm$r~ zKA#LrezvPV=qLSY-Pr+(T-98POcABo&J13{M9R#vqxIk8-QURr*8do#Z6v=%f4HCa zyO8g}jEA&Fnfph@Rh1%y37!F^&=0-8h;U+G?lms0H?lmjvXq@vhi-ocgHf3Ny6K=yCft@^@dm(0K+)Jj$Lzjz^jW} z(k~X{+FmLSElRq=$5I>Budk9d`pE%3zZ*GNsWpFD%(Sctft!v>_b3pU`s%EZ^V+^G zvK^&7ZoABa(?h8$@A$NBC1MbTg3@Yf_b$KN`JcBP89clbYR(RZKPbZgo7RJJ@?N#Z z?Cpm_g(?TtgSz$kd5rziXWcYWTboIEVx=uSvTDYCzc3_e8(}7HFxeQ^yRKXE6#JcH zOkZErvwyJS5x|--`j#CW*TuDO8*}4k9m)Lah*rnFctE{E;Rx7XKw-Tco&4Wr^msJ=K59si}+@uVb5|3)T9slpDQ|M>tA#4tuhx<$4zs4rD>~zJ!7I12@8=HM zATn=__jeoHzB5dVv=#rn2teCbKV7?{Z%?auyU{hpLoLII!j!_*L5H|%zr_J1!=5s9 z`M?HBuH#OJ?CQyZhcg0Z<*Z7fi6trh*%_VIz=@Fnf^ZQ0BoS6{URy@=*CmbT@hoHJrVnvUcD5pE?i z+iM>i_LC^gUr_gOb8CU2Gft4526pLLB(xznLS3|et+xA7ALF&5E&ViCbp+h~DMgX( z%|9oRtm~htvNq}a3ovd2ac{EwLfl9A2U=A&(JO~38aj_9dTg3E0;)f64U;{b=xPBj zeCG|~Z?y?i;bg)M)Zvx(ky{_8c+4f3v@Cqh`EUv?h2wy3EzZ*CL|vB~URt&!bT5 z;N~y^d<~@P_+t9SX*1tX^0=tjPT|g7;%hu)#6l#_Tq~3zl$*<|{Z@yAKW=B=2h0pq zFo0~51L3U%(s9sO_JS{ZO<@5#&zRWP!BRsfrc)7*h5s&KUsf!^)fpkA`-tp|g!7DS zBUO0vc3)+^6NA}atF%_GtfddbE9-=1M$NIp#d zflkby>$#6JE^wbXs2$I00#2W+8AFqklTxSA8>y!%Id)Kt^>V(8p728|>({~>D>Sxo z-(!-Tw`*b(%3M_{#`XVi&~qJn{lW{hTQ+!;06T5~IVY&aUp|ras^|peoMY|&$rI15 zc-p?oC!_BPt5jCJkMK!}dK24OFpG4!EADz3OrW8LGf#=Ue`Od#Y9rhd%*bQ)TGr z=J3A%bg|T_EO7iAMuq=6c3}?u0iUBk&1o3(H5N2(f#6Nz$VL#tfXc$r1Uxu!)d2>4 z#lNdT&_O5UPP)f5625p{a=5UKp*MKzLU&o)k7V-#5?0*G`Z^}4Wm?}$vQBEJ5mKw#c=pP8LLhrt@FT)L48HB&Iy9GT<(?Z(IF2ljJH+&dotl_L6OT@*^2-ubV5VdSeZ>~VKW zz4|RL>sOPGPK<^1u$(el>-+$zitwuipH$(+krj4JM^fz_y})0}KSf*jSEB&2^6)xM=Q1X4@OW4~j7rSSFD_WH;2FW`>l3Ojmr-ssG z@b<3$G@>iDYQga$M+q1G`y)270C%NLV(4{V7`4K^{yU(yF=G#F93kW%N`n<`$(y>3 z?po#3r^K+2;gir_36bZNnN0NUTTL_)s1g)$vy%TV3$HS{zdPL{f5~UiEMq2DPPN1j zdnI!9RaOu!`HO3?hDfxBv(Bu5EBY>@-eX*4ocNjeDnFTtwVQI`N)H9#fo^_VYFkn} z=|bIpZZ@f6a;jLvfAA7FszDGC6#0QB#3KP4sPuNx!`hlTLxWL2vTxOV&DD9UPOtze z1VLvc3gHE3)bqvVJt%BgQ7@RjoLp)<%x7@Z{|p>DfiddG{J(Gf#z)bWMU?epY;&hfw&FD-PapZ>z=4- zs^MNzy;QzDfp7n$Jk&9PpW2=}j69ZEhGw1yh)U_EG@=Oc^Wm%QAF?jgpvYCo>8?Md zWA;CCM&^iu8IeMa8UerLcdWR%xf=m-KvcVoe~t-Nl+wZ!ivnRyJok1&pfoC9|8X9k z-YYii-Zu2Ws>k?9(!*B%$$1<9tO`f^yAs?z^1Y?e9R4QR=3a=20pFkSW2?Ui)l#3f zfk6~BVbd`8s6A%##Z-@me`fhR>h}xpHwky73(~bW3AY0P8WBAd71B!=E4YLGs{0b_ zAA_q7Y4y=N7&k)Jsb4Emy?z;S#l32v$c5?&q$Gc@T;2l=y2xN)$Q5t`?Yw+iWLxNO z+H@W*8B5mSqUaY^#drK2nv%03*3P{3FJh11B*uk!$#M z^GFMw(ONAf9c2W}TSrC)gWM1J5XeD${LjIWn*mtv8OVPFLl6YxX>*dso5#3|lWE24 zUVnaF&6CGuxQ(1x6%Xz?lm#dT9fpzekAa$;!kb2FaWl1Mb2$s_BHXlkk#R)Q?q=6r z92eMx1gR`IEKn$oMDE(!x?aI_cI+dZsf`T^i1|fQ7(lAn@TAY=A7yu5-`biRNe2ZR z!5Xh5E3@

`PDfGua&3=;|Tzp%UlH!005hLW4q+SsNU24{=>_{buNiTtXRL#BZcE zI2)lK zazVyC!61iu^5EUaZvo_CxZv+Pz;!yc7styQ5wErVZ3pNQMCVF~$ zwFO@Fm&R)L;%C{3=7zRA=Ndan6*+;F6U4a~(DeWxtUhqzYpS=>eJN0F@7n|_dN-KO z&1~-5d)@XU+Tr6@u<)V*{_W$dt5%)Y)MyJbxck!!KatK`%&W!cs(;8o($l`L?KNF| zS?0DWQNFpI87YH40~aG=K5}w#3d#y*PfkgJWb*XiF7EDOU~qZ(F`Qk`=Hb7)&wLJm zM=)QA6(f$g#YNkANJWJI5zQ?82J}>y_;a=&In4Le*50&F5*q&e7==Q`-IkZpBKAtG-*L!&mjXSw4Sq?dy-uvz(l9^I&xbF*5jnL$k-uT3!|ExH5;>F@dMr z%Yi}JV2VUyh++GOxvr3arhPa8bJ)OQ6pdl p1UvwosLJzVgn#+*tGU~!1VW+v7O8c;aD6D$Z3Pwi53(jf{|jpcRLlSX literal 30688 zcmce;2RN4h-#>h6NE*n<$X~RyIjg5*d*ZLR3U%NKqn6$W~U_M5H1km60T} zm7Vdt&VKjr-uM0d|Htth&u<*xzHwdGd4A5%=ly=I_jOZ8TXhRHD>Xq7TaKzJ>k-5n zcY>feLq&eBmO~oMnhGZSSA1auq-v2Ah?L5%7+a+;zzqZ&B8oa6(&EXKYws% zZ+p5y2bDChW}BRPM_xc&$s_Sk{3oi_8l3l@jy-D|6i2I~CVp;n-A9Q+iBHMgn)&`r z>i6%ouBbe~prD{iX3J3Vy4#0~gOhnr z;ZMw#jx>VqfB&HFA1#A_4;AOHZeS;WD|TDo_i*x8G;caN*ONbaqP)L^{0Xx@Pkfhn z9cxx=>Mz}r_-yEstFKnFsFf((r zii%2m-IsD7uLmO%b)5TO%IxrBxA$OS|&b_@C5G{DDv0YbJ_wZfOlg&>J zg}o=MHqhQw9_lHRAs$-2xpPZBrF}8$b6suiI->dcdHsD3pBaXShlf5F2pbq0+J%i6 z4olRT7?SJIxmsk~E_6F4=Jfa1k!qTn2Cn-<#b-^)waQz4UsJPIUS9rnV`Fvh>6%;7 z(T|31FJDgSODwPVLr)zzt27DsYy8?aoA zTiFl#3uxUlu-i^cOR%%EZ)TI)tQx^_eAWeaAE|rYM?2}J zrl#75*$=u>$+(Z#CdhexZ*=T_vEJ9$H(uU{6>oXt$rAyQ-MhsvbnS176;~>jcC7Z_ zIR7&>(VX@DO?p9ECMKi77t4z?UgzcNOdf4}N3MOlF1A@%qDmFBw!`l7C`uA`$v5IcA7JcNrUxTK}G8yOkVhP(>i93fse%t#)J zITbZEDya)yL8Ya#+^65Xd82gX$oe2PwIN%cZ8vl6KS_uj&;Nch05?fpMPsn1u7^$f zLf4^G?FZ)RVQPGe*a*1H8@i8F+V#(dig%ovT+4_zRjm(aZ%NVEo|e`&W_mw*+D=M@_nEETPt{rO6xO?r9b*Om2w^XZ7FU1``?+a4q>6_t6NlD2q zDhdn_-+Jx(^%4F%smBt_XRds@AK7pgwPwSP9Xpz{Ps%8(sGOM{{p2w{L`R;yNM2P% zt+9nyl=v*o`2BRYwzfL@`bzhu9B8+1ug7ABUY#hhoOr!BGa`PbHptjkN!jhP&@aOs z`<)1GZf?hdW3%VTgR*`T(?ieSKVq@uD4RUIypgf7K}SNEs+yWI>n_m?Xdd$PJRraD zo!+e2iDI|ubC3GSy>%uvwR2s0?#V}^1#)w9(_FfThK7nbYCb))7QZywS=1xu<%Y|e zU!LpDJ6%J$L*~+ECMKr&;f$O;p1;mAv$1_lrx6www*QpB9!nJX^r@|!Y7?{N1dVx# zrQ=)2isc&-@{3farl!@LBTGw5tRox;oY!$)ndg39P|%WNq0sfhZ4;qmWF+9e(DC-I zR(AK#*3{G1)*H06w4OVBVWOj>bFP0Me_*YbmzRy5T@C8xg$oy6zj;%Wee&g=Q=8&HJgK0iNy{dCO@ZZR<$ui4S6?w9U;BO@mUYqLK* zJ{rA05Btv-yQ_?`Vr5ZSetBA-fq~&`)%6XNzke&8I>ohvon04~owf2n#+{atF`yw@ zu>X5Q^icOpX^Fjilb$@e@zQ;w!{fj#o-0{G$xjOn-Q3yPd1`v7W&Zbj1q!02rG@-a zTU*=Y*Q&K@eB1nQ+^Bx@W>a={cC3PLd9$&K3S|(L!{_Je717n}^78Ux54s68y?r~? zySlP{E#>Az-rlmwj~)$-r=Ty+{y@oh%q%V4@yz-?CEe-H+|zrcr0R+<_LE)Cti*YJ zQc_a;=cap-o3`CtNB$1+I5pMs_m8)PUq{x7kHtr`0s}WRHa70TZXFsPuKx6iMcF?_ z*mw<5T3V`OYDz6FEv=TK=i%XTNJC@OlP6DR=Esja;03fC9O2{LFKg=RD7TB69m3+E zDbj7*b}gJ;M(Ch28n$|TSDy6jM+@J+zM4S7?}^s^H6AN*35$!1gx~cIG##G{hqWg0 z+O&cm95!rAD#FRh$xn^%+OubS2`*95tq~k@+ddoj_$kqL=?fdRv}Z{NO-kH2Hvg#9bmkB6A|{Q15#1cN3`c$yS9exkh@P5)|Ptyo@@|I z4`NEVk{RG7N#%H*6c8!zHNW{bkd|7I_Jnfm4TY=hChdf zs4VL1>dN`f%u2=Fy-O~6Zhn6CaC=7Ij~_Q4KjvGStf9$${@md2dk&HBX;O5d+yBpJ zA?XdGyLMIKVG#tLcUp&vKs1G4SL;(zQPG0%)926cl091D^XC$<%4_zCu99a?vLYgy zq75RVqLjpyr}-Q^&~~sv>zd;BPAv>3_P=`|V^ZSGuc@iYz|LwJDrrhS7(B`w=vHey z%8xo>$Ds{S@hGBnXE<1|A+Tv)zkA2@Su1|O?)Zmpg~U6C#>RmW5i|t37#2P=jQRQb zPBX)XOVh2#Y(6~RXq08To+$QRl}C|`1YENFm|fM|%SklnTWKU!+?RIb;o|xlaQE8r zF+DxKucM=(ID<)8E^JiFjvNcIqUuJpufsukMSKzxQ7>G7DgqsxJ$qJ}UQqk7;IRa% z`ijM2VN=`%`vGTv`NbjL4;>v(q#Qms7xiAz7#J9E_Bn53LrdI~^~^W(JamXCux$@+ zYj022x-a$o+*{U?LI8$TfVz$K6$?Mugm&-d=Shx=+IDaMMV|9^cKUy2$9|4}dN$)B zbz2_`yng-qNR;aWhc9$QeXg`XoM{iucH!Mw6GrFtZ=W^yTg zD9^I~8@bfaZRmcyPo!~kb2}3F3U_5rw(2c)DClJW0mx)#VWA-8=fAF9ullO-&6~W- zGq$J>YjHq7=2+Ax9|`$ehbV)K0<5>F;It>mdR`-OCQdoA$7Ax;de!z~Igd#K6{FaD zLBc50^R zeL~-d6b+OddBIC4BphG8#+fah$?Nf6a2ByB-+{L$(cQd0KfvQM-MhQ!i`S;7U5j75 z;5m4ZRbgpzEuJ-dy5h{TE!0N=|Os7b@wj z?9V7{^w4*81qb~Qn*KjNx&OQN&CneOw(&#;SxhttOGxOo=u>|B^od#Cd;c9Vi}1?I zL(0m^hm4JlTTX)sc&+t5Zq(5E>C@?iA%X?|6 zK6vX*;MNnB4G*I7a&s%wbRRLWQ+jPYBIthl^yyeB2UdG~ds;d=3VnV3fVvvo5P;o!D5gVEj!&crK=>B&lGLdA_} zJO;wW`HjuZ!~OgSTwK-b*HpgCGuz2qd{X=OhfLeIZ*X^aUz+)7VNz(%Qd(A)^z`Y| zp|mG9WGW0@ykG962hO)nBBkKEU<1pZF;@-!7Xay?3aWK@)% zqh*J5oF*kusfnrS8Jq#Cty}A`v8Qr;mo-|Fc4eJ!ebO*yVPatLm^=buI|SV`4uj@4HG zz(6GtCUR4|489u})9T^I>3VM4uHJHbPhG5^@pz|3f&C}F;_uTy(BvU>zMCXOfAi^p zxw$#v_xiP($%zx>LIDELqap@e;gt6dA`zBQjG_qBBbES`Y~|axS0Q3M{?y*22Wg|)xbg70b5EKroSh}M(bKnN7>l8m z&(F`F`cmlNu{3|NETPwUYWk->iN}DHuid=4adCDmq_}uL$vx58s!_S#+&szdHr#gf zf;_f61#xHh34z<_g)~I2%b<4WmoG}HsvA%iRu|s;n(r3_dK_5PR#$&2<(tv-#S>=@ z`|Pm3KE1=2!XwJc{!XK9>H0%mC3~l5X7Zf+6iNJb^XAPNS0n7+tFa%PE?p9^wjM0% znZScO|F!b!<0ns4&=!@IZ$EgTH9I>?7F?7*#covQ7Idogw6vhY3*G8lw{E>18>!h81IBAOqmMg#>7VX^js z;sl3<@yBVZ$a&5385$a{In?Vj$9}TZBNA*}r9Ohw_h+ZFGT5;Io>9r1t)-fKwAN|(y;*R{K$Uz?p=8K^6Uw9O-=8S zJV~ax@oxHey9gW?0iNXidXKrd$^eDZm|+Pi-I?#%xw&`>DH?bE^YJv>?VRVZ@%6E<8>*_V zk`2`MV>UaU=*Eo%A+c}YpFlEFPYFsG$^Gc%n(~6ryE#k_X=0eZ%jrti0FyUFMxGLxEWI9%bd{ zj=l104k_K(oTPDAl#Y3W?3H+a_V{hmF8UPea7=uIpC#KV-#Pj12&mdT#>u7 zvS4jvYipEI;raU_1qn|{1u z=L(Nh|law}qH1R@j!iK6v149(=)^E zCZ!(Y=ibF{L_u^N?Tn;<>0a5|N?-X~jz-L)JQR=j^y^#P)l;By6z>z{NI5|sA{!qc zA632Wy_}rnVIMCLtBfu)t)oY;nuyea3R0rk-M(|@%%^;-q1IFyh%h}iW_7X70PyXf zaJIRl8=R(xbVs{N=+Ql1p`ct`UGXt{;kpt3`750rvasN+{CzfE07sf65as3aXDluI zhKEi2pR!LbQMzx~u%YRJOp&&9RE7QM_PnAZ)iY;!)~sE7+Qufxv3!B@@ZrNbT{K?4 z05;UZl_ah~DSMdwtHL4-cRf6~H8_+O5{3Ujk}aG>Ml z6w4G)QC01~?^vF$zd1Co?q)Qe+S~ims)_PGli+AcO_#oaJ z>#2tC*tLU~2SQgLo_OqDYwfVRi*}E)vUYo)Ja7y;(7-V8Ky_y)Gm#6a#5SWApKVfL z%LpYW4~H4NVngNcHzhzlY}(_U!Yn{aD4V=R=ST2(@~qx4dCz{bR*b`iqNCmT_U#P% zVr5s?4zw~zS~aNLr_Y_EBsvT1w&RA$T^ujtzBM2qpxE=bz?IcycdR#MngL=w@)t1oUV8ts}l|1^PkkQLHD6xmE@D%0p-EmavA(2G9`uo{Q2{`XjVV5e%0c@c^_I^AK{K#&|Y_m zh=kUj5r6wY#`r=AHSrwi-1NCkq}lQIy9PojB=A2!+9>FoQ4Trrs6e!?lhYo2PGnx5 zc-;N_t-l7|WXL7`k^0JiA}QB>B1Eh_?P5qI>PMuB$P@Atumg61azQOQnxc14&Pz%w zLAKYqM>vQ`U4ZqeYZ8&lZq!&%_EFu?-T7YXxMxo-@SX6oxp1qDtR`(OH%RKxwnd7y&pb&_}+MPC}!^U zrmb65w6))T*__j}4DmZw#$6Pmk!M8?fZ680ZF2;xxF*2VxRy>}%fK0)~$;p{?(!*oFq{HXi?-JxRP`3BE{Lox;sIRXNU=Oy` znnPeSVxc4G-j6dfG&az1Jch#%92CTrXQC-3C1qf0s{TJwft(Iwz`B|7?)yYDbk~4i zxG@soqLu0cPJHU@)B!f!wR?Bih2C;4oSU=O)`D$!8_urt12BV3fRAnio_e}?jA!rO zXl#Rx1V9%ZT?;rb34O7IyLfr8tAsKi#&%+0W7AMp#x*@h6N+j$>k3juqT%PypW9BX zB?(9AWT~V8VLlIalb0ZG2OA;<9Gbiu{J2@cH>~Wv3m-U7?U}r;KQ|*H1c`k(1|`2x zWmqNHTFXm!$EHLDPGS#s10)Mh@Jb4T z6gAO!v9bZpPl}6~6A}_k^3JS>Q9ux|GL*3T=!3(uQgo+(G{uoZMSOfbA-kHE2BnQu zN$cR1(6O^`!LyqG^9NLngG5T8vL=?kdg%-!A9LzKd3F|k)-7@V_TvZT*9F^p>qH+C zi^bl%muyjSh1B-#!t{9}NwucfZRC#5xl^b7(2n|nvI#$MDG`%g!unnOVK@?mtS>F- zh9{ETE z>-Uk)rGZAqk}*zTMRCXzr2RqKqRPs(8e%`wGct^YTXDS0%P)Z%s!@n8^3~IgeE%EumVy8haoo28xIhqi+|C2%&Yf%S=uiO`^c$X&E^7i5+GF3j z2mRk+oHyNxc9@$s8oTz<2Tkd6|B zMU%42k{#J6L-2c+3%>^1-#NA~jt&$Yb9?g7EP#S9st%;3!^XzjflPNnTMxXlym)$| zr>tPbEXFF_K;Q~YFy{drF6Vj{%gk*#UAsyeSKB{cihCvnen$8K+e))m{Gu^y*Ek*n zZ=)F+nja~-08by*(P?Ij=w?oT`cx5T4&o3qSp1bgW722OojZf-Jk(pUszI-4701gi zbICs;fdfZ;pb^6OS^}2bN4EO6%=)5e`Ug&&IFXi?b_zHP|46%3>YS4F%h_O{ObfDG zpoxeIjurN5FbEbG>>;wxB_?wEt!5QAZ<6waO(_hy;sy%WYdNjh`1nwe(W+Ok))W4n zLiN5Zk!tH8u#G~zf%g5%eDEyb^TtW_qjFnfJ!GU`CVg zwVC}O1HjblhK4HiT5_-SSsQF`lu(8IP1?)Ks;X}sn6Hd>irbt&PsRn@X8@uCBRf8@AEXLDGYk zO%0?#s=d&aC?~x)F??Mp(>PIRe;DR-!4%jtEiJ8IeEbelFNTkfP7*8cbC7cD4k~MF zYxrtZ{j0zA_nM!D@lEb{GqdW6W0EX1jg*(ZS)UiFlDU4*JaIzOmq`+XMo0kE$B zy1=>6xIr0cmxA;uNQWCpY*O_joT!(V#;WtJ-sqT_ZAR$>^HxFuLp39nWa1PyKPlT? zxF8DFaab92^Vox4eak0kjjF1uq*{itHgqNc76W)!w9TY}MC%P^lN>h;F<3w26nW+?1vHNEuPSSOp&&OEXg4u445%f_x`?gL^y|>X;#;lFZZXXr_;cao+F$=y=fCK zdM(7#6d;W^Wfm?jE<8L!A|j99E3C$2L#3mOxk4s4#s0#f*0r(OIWRa#PuBvM?)8Tc zQRshZxDMBEwHtXZm%FL_7gCX)oSbax{WVMHyZ7qyoa6(EEu; zx6h`Ti!^Cw$G)`U|8|-6CzAGb42?0-O$FKsg*3j|iw`h#;v2PsP0Pc>;~{+epIAL* zA@u9Rhg>4&rSk!7*D6Bq-)Dp3cX4G2ZR+-zH*z5if8B`xzybc|SL_abp_nmN&i?j2$KSRuxYoNXVTg2!;GK^lA zRONSx3K8GGf4_U>hkzaPzpx5}g`c#tgDjS#zWYn`d7wWApd;Ma%Kmgnr&D9y`t?>;Yh_hcUA-p5DC=_8?0;t*lnv~R%a8uo%mGevl)Kag z9i3Z~EO|M9KIH@7Ub=K?X2&keOm5OX)zQ_ZL*YK{=tx6PPmkh}bn@jT6sjX&Ek@a9 z$FsEHLVyiI>ZJjxK7^wNyxv*nC7o?i5e{Bh>ZHkYIm_qG|KvaXYV@Hv1hi*nX7(^O zH4ymiRLzYoIqfuFt4sgJS$Y?&3xoe1OW{sDGHeygvurpclkl`iob2p?M@c$V_Me|`BrVNt+qT`h zcP|(~on!FbgX;walIZ((?5&lR1gU?3=m^)dA@G94)MnOwRDdcgD=Qw0)4M>&ry6~L zGXGm*GaK{g%tRu@J6=&y?UvK8r)FkeMJTKU;iyj8?z0>X@JN&~dW1T;dB=WQ!fEQ; z5kN5NdLiR+L^}kF)SV!Aku@GfpxE{2HfZ|+mK_}(aQfGNE^>^(&2?xrf%rh3@D=s} zL*a&nA}%hzXU`s>{oUu@Cxr29+^IyTf~V~*+@E{n4GfVM?$@6R9oS&^aWjQ5vQt}T zUhR*OG2%svs;k16^Xk>B&NojQLdmWqh3|<$m`$Uj_5hP__%9iHdU#Y;S0~q5!YPB5 z=H%+?5Buq_H{8?HvqxO~Ff0KkG4sv5yu7h-aU_l){NUss;@ifJ$QGbPwT~)(PVpYc6PS)nSlBT@~{EueL#wpW1kHz902%0;oIX6JhAyps8&b+cS5B< zuE4QcDtQR*zl}w!36@3C6LcmsD+@h?0^yXUrQh$X^J;6Ar-$3gYy{X_Cl-W5;fh)6 z+>@tIskOA;emp3K=%D|V7$rRE>xp(g#r2A>p{_%fD zli-jP6$$<|Jb>U`0XG{P8o0siqty7ofrOqX%6dw$u(CGJc%nN)Xh{JK)H65d0I#@Z zA|k-X##T~j_^Q61o?w-`>{>8K<#%H<%P-T393cWNf2wYI-z9YZ>Gu|vaBq#<7T{E<3tVsh)jg9kbm0?~8?wk{o=+^RbX zZ%d{Cp+Qsb5IVR9IyzW%F@R}1#l<6`EZcx?gG4GKkfN+iz9p*nf3#?Ld6I$JwuQVB z^L`IVvk%6WX#^a|I6TG??{auh*3vm$dA4MANKWQ?ZvSZ;?9cXaEXbRzP(-CP<0RID zAs|^5c;iN$T^RVPNo{@on-~D0jfrdreOJ6)$38RSz2^}-h>VWD4ye}lne*SXATn%- zL|PT#(_JwOk_Xw@8!Yp7f?r>m~5?|goq9@(C?*x#h$heal-4-k?l zWN>JniqK%g!^2GwWH=$e7HBIRa8gL!}@dq7iqLgQAe}7&|+=66kR5MB3~gwG^ZmO2mZ_Em>Q6;$==m zdm_M563N`aA_MY~7kKSJw~dW0gk|qJ(itJ`6m(hh{fU0}d}j>@2jCFgN=OKYm_)TY zmG1p;5Gb(u!v{qW%$C$+ob!u|JpBCYAuK(e>DU1GiXEvD+s}m#n+RxZ2qLP4xnZqH zAs2}3L!+|TfddO+h2c87>GJ$`LKz|0i39;4GSK+8sR>Ld46K0li{VNV{`I!-`*-i| z$h>1MM8K5vcu=nhzsX6b5}$r_3o_)O`QRWUj>CEkJW7h_cHwm)}I@FYbY4OUBNSb_FU$ zpT9Ht`22sSFy%2D7=<|bfk5Etvvm1`UhJ09DY%*UC zhGEl@$wF%Gb9|O?pyAR`Zr^^F)e=ph9+?_icJ|$K24g6h4^vVCAZ&H8D%kA@&XSdr zBNcoy8-VB^Z8%O=A9i=_zVlICJ}HLAssvFy2LUQxqrBSDp#n%W^CF8}r;*ez_xrAZFAJlYVxm?Xwzd~ zwr|=lS_>UV7d{1Q5$WjQgawd+jZbY24IAO&)gsE(oOMFlzPFqmJ$L8s-LD|Ew=FrIu9hopzv@?BL1KowX#2~Sww41(~c>t9X)y&aj;*%e_sT` zCHx?u0i_@Yy%q@)@& zT8<$3p%W+AE?&IoEL;#%!~pwn6G{dd8i)56SY55yRq9C(v4`mg?u-HWlOV812tU9% zupF*^`wY~YkTl69@8U!udD&#Nprqsg0qh_IdSs}6AcoIUVg@x8?aUk6dj*my$sZk8O%t|~;Q zETi$UF|Es&FGJ}P{ETca6b{mNC~+D5fV&>}5sg)#qhn-ZBHa*eG|{#^OVxvJ!wgv_ zdDliqM@c&Y&+%puawJ-3&!0aE*Qc$_%l+WRZ|msfdd>Ow?~er+CG7?zB+rB386aM- zEY?G9)l=%({$l~_lhNLvrNeADO}Y?M8wB$&>wA{uI=fhOINem{fna#5EloFg>p&S3 z3(JBLhri8Em;t#SQ)f!vl;^Y;U2>&L_>IJt!vAJu^mxS#PP%a@+Jr(U4MXzv3>y4LHuUcZ$G^^P*y$h(cu%QMXfPjDi*>FJGzmANAAQG8rR+4XeA5a)N z`9tg<62dC;90q*M^P0P0R&j;ht|Qa4FIIWl^v|7?$B!$kswjZ0$jEeE1ZM<_2U^xZ z%%3%Wh)7t%Gbdx)va+(~88o=u(UYX7EF>;YS6Em`#+6{Lk{L!Q$o;SYuR}SKUz$|f z0hKgc3xdZ%_wi8JbA1^(c_gCnKL7)iV|cf~J{UD1-H_>@tpIBJslOlo15zJsx5CA-uF)Ph zkwil#EnuYEV39gniUD6 zBEBL-J5l~;D}XPlZ?61KT>T1*mdu+$_SH2qx`EZZ9`rYl&dPc%=#VlL%F3z!dT|J3 zq~Z=>S_O@goM50>OF5ZhL>_6e1k$%8jelb%Mn-<`3qvp;TMF!q@tFEsk0~4hH%^fC)U)$d_K(|ZeP(HX%E{UJ z8bYNQx>*mFWmfLZ0=521)_xXjhZ~{)HvlmX<}UI2Bu9Hk2PFa9r{l9R@GT(6X$Obh z#>g&iCE7VCVXW5OrU2l$2S~Aoh>DJeUao7&PKx|-aTEmVEFa%x(6DO$_K!))dOqqM z21eUyIAjR|l3QcU{kXWpXncq|<+O{TPIcVE0MXO)hz+5ekvsj?ty>2R&i9tfkvtTq z7Ek_30&gZ8#3rH=hw&--QA#k9a%=6KO0bd5~V;vb3%UeU_p9Uya$3e1?C^# zBEgSKwH7~vga2r~7?Eic4@ipLRJ2VE2dqf-)h)#kjgB(BM2NHiZ2r>C!0E$gBH>m(xg8ns+ zu%2jcZaxHC7NquHGNX@EQcfe3z`Wmy3nHV1$JQV!tnaOcvzUZpjMA(V!m@1z8$~SM zV?d&gIYSJn+jIutiG)g!qe;kt=Bosy9$`i*Dyqs@am&fMUf)eL(nKXr8y!NXP&2*Z zn;>284_t!}0))uDvV6H@(U3HQt2=~!R~7_UmM45IVcYCSo{b>q6HH!S+6}_zFE@^4 zKrp~#HS{DX3yNJZ`r%5kN>Y>FB~UrkeLG|ilK@|#msV;he6xo*2jZrRVI$a%R8aci z1DCZNuxd>CwesPqOL{iWy^)3Tw19iXGfFGS&;%Esd}be#bL5QR-)wH9*KP@kZAiI0 zPSYNQ{j$B=$7n}|K|1oU(F2K25d7saC&$j=Q4$A4f zBo4}i&@e*tJC`;xs{1P>eOs*JGdJ_#Njf|KOiO98W zf4ayKT($v9ruf;{wB)b_23Me$>=YK(cQwI1#(wY+6m}<@o6h}o!Q@dNV6wn=oc?_h zbQencS^@z)PEtNnfLBV3Le61~D-`DcHn2P6^m*7AWbM*KPZ&CL z6+%s*TGffZD;tkYNH6ERn0RyGu=kDLO1{OU#`+A22zi#uG_P{F<8L=HbC`lb)d=Kk@zN&!br2 zBotU@CnqlCOA!$LcSgIS8Hg-Db_hbRkKrhh8CL{%b=mUj56?B?br~F@jYM2*>@AFCqQtg$c6|C2hjKBpyKwu1{_V^5LI?xJ0e=iE zZlL7&CnMRPgql69WA&J_dq4Oh$S#k**ho5+Uv0sIU2f^Jt z8skAhB%dJ26;=F%tmQ_6Ur>-Iw>AvWbLCo zZ?KWRv$1~i$0u5BrY~G~L54qJbE3C_UH9!^v8V!D@^HgR$lQoNQE%Kx*9d7!8+PV% znAZp2=N*4`u7QlR0>f>Kl1ct}-(>~>0nd8oY>}yh@MWYAppjpNh^5m3#boH`&pyPh zg2KX%AYX_u?!f1gmoNKCJ04_5+-K2ME#-r)4^+Li1T@3|2?qxU;t*`Huqm^bm$m@O zz%5lOu!Ox#5PxSxaIb>yOGs!HBM$E1SaA4&@5<7qh`YCM69jtabSZg0mQVYk7 z(64U*Fjf4D-EwlEh6(B*lJr8Mp`m2-_%i#mCoX1p6ScIUqz`;v)jv6CafQZrbQQ)?RLTBJN|Z#s!h=^K{{TamFW9} zqtb^qPCCd)00m>d<1~V*&c_%rA4omD8h!#fL<<*i_(82s>w)Yy?~kdcpPz$>zE)S zC3=X4hhb%7Akzeukc*3p0G@Wj&=H~oNd4@=ej^`N$EnhGU3$4S^GSsv(M>i)Vge zp$b_GP%1LjOo}pt2hN>8zmY(gi=4~`lcuAmUrUftT&R>5cJCw~AA3iBnZR&nq3Uj+ z74b@3c9i|u)Rc34r!={ zmKNP?`?vQbs|y)paWehu715{o~2O zaWSW?lw4w)Men8mNsU=%ndd|^79MbrW+AvB*)NFpeHsq#t6rt0m(PN zH*-5=^oE+-KaGQJ3q#71wGMU~#`U)3M~*1Xg4*PrZxyHuXMaO~Og72uy9o=gYWTWl zhbCcL(Y0&WLTMvoo#je&id4Eq_AmNvKOx~R@jJSyGgajK-vVSP`bb*R( zl~`&@GIpR1pFmtf^46&D*<16TJ*&Y$iCFo(VsBCqHOwzdYwOg%m)~|+Vo;mB*yOE| z-~YEi3~52q1qFJ91CJ28ac>{_Dz|I6R7d1ItB|EWaCthd^t}oGK8+`;YhxLU%q0;7 zt6JQdaW~2jcp4rw3j+-J1ajei+jd2D-Lx%2S`FU9HK2Pa6jsR2zsUb(9&>4V(M=_M z$5Di=#W9{+nZh$EYi;%0yv7UUl0?bLkC#_9QyrPcc zfr0t6=*bumSO|9`b8>hVFhu)tsgPhEkP{BAF%fLiWOM|!#B2Rya>tb_$CckPI!2WTaQw4U*Mg%!`S`xVUSpCcyI;n%E5l(r^r+>O zrR8;%M~5G^U`ozUJ>`SDO){wo^Kx@vh2#BGhJo9^n3n~&^#kdNbzJZ{5{BW*?L~OQ z+;f1Dax-%!u*n+A@A%`ZQE8s+2gYpSA z1Dq-txvXcnR_IGJ>`#f%urRzfy}Sln^kvj{Z38yMmI%zqTKqhSq=xH64?EmTR79li zOE0vIGa;i57&-qfL@f8&Gs~%Oudq*zO^oOn85lqv6tT3}OntaOJfj`l6zuh)7y?8KklCO44RX-j}K}J2I=-9 zc`Fg(i2;hKU%#pw8-v|@JOgpB;i#+BZ~SS$CG1Lm=kX)X`nGj~r%LbXAtV!a!Ep(O zsVw2xBg-!?uG`SmWB~i(QR(W+pWx)3>gUcqS+f%)-iYCd?iI}Mtk(gcPqiD2F*esl z(ueVg?rli(R)|f%$Op!KRa2wniBA|eHEJc)L9tWby?yu22=l!unhCPU&v)h~t=Wn1 zg9xsusECS*F?<^-e;KkaLyUR~dd)7d-_&p8T&PMK9pB9%(yPoQ3v6Z}h)!S=9u55CP&26Luf2#1!j(2CUQBW`- z&vitXVIZrhM~{41IE`e;xe?|DBQDqqpanoaJRg$_R`3skuV26ZZFDpfV@7JwOLQ^v zGj3wk>O{lq?j98%pZRxm7kP7k|W8re$jOIQV; zxo{yHlNV+*4QeFuuAo*!iObjxh}ze@TXtt*x`* z1!KmB?%mkkjn~$1VhB@tv|mO>81*16>JsM>*jOLn{0M!vq zi^0|$c~=sX_F(@-#f#3)AK2$+W(?t-lQlKSi!Z_|AZ=_P>lPF+K}=R4l=lmE_3tj1 z_YKrv{yc24aZT*~9F;18Q#;g|$BWUFQ|(U;!4Jm@T~O4kQ4d&vPjwcl+;N&2$)`|; zLJ)MYx}_x&v6@_@XUS+cNEM@Y=>v`kmvNDSCN$CnDRtj66a|wX_BxK@H+7R-S+yB9b!vuEKeM zM1e1qMWm$eU-z%D$;89qgXp`ipDXg)T6`QXoF7nKx5tT zH#x&vv@1+fFD);7j${?>_x#02rk3Hh^(M7@AWMRQj9bXHOhS^UXJDuWh4=h3Rup&8 z%{H#~Ez)oxDNi)xs<8(Tz$~Jmpde?GmPRaA*Qgsi`A)Z}mM+hJvDwE8M-&q_XF;bl z;gm&1Mahs0zPv|=qoDV)Tj%RYUJ9JgkOW>jMnq}=v;7b+F#J=1asr^kfMqAY1EILM z*x-+30IlNCTLDiIVP*D@aH-%Lz`=M3k?hxvP3Z3ZXpP_K@a+dpD8tAuJ%ogh?`lx% z<=%*&3Pg3ai-VtJ<-c0{OV@{Uk?bUp`aBWDWWg7Pkl!q#u>(!}>D?ymVI;F|hK04e zGbcxJ_%2;6Yw{Hq75&j}Y+;4X8IABxmQ+yNH>M5#nhzA)%LOn`@9euD%d5mY1kUuSfMF;};xQB!F8+!-`e?})q0g3oo&fa7F;bEM^Zm%9# zb-m^i;QJBjRTDy65HFHOt9W}DUj_x6$g0wh0!=F)5@}%hy`qctb($u7`;yG{yN=I? zJDm}z8oClHNKXR@Vl}&~VNu5;fyja+AbuS1nB)R!A{!${ zSXBQ$oZ=0*aqbI%fAz2xSfuQkG0xgoHrMkAWbPkJv+n4)Kka~oq@)$b-o3FE$^C{D zNd)#j^+!0(i;-e2@CnMo-YZKUva+&gdCj=}J=FeQ zUNjzt3t~wKqP~M)OU}b09#2x#LqA`4@5bwP29x7E_$sVq@CSSckfue_Chl6^!`K3~ zKAwv|QxCY0??4U(ar_$yD15+IfdJz;%FK1Be*x$xYf2Cu?*+CgAW>b(zaJ`%yhMW6 z_yO_>nV2x+>xha+7EH64j_8uHLNs)A_c(ul|3HQ;KuuQiH3m1)^+{6)GRb^J42XVk zNXT(mLL@N8sb*kgl=I&iK$|=s?G9G+v`8asnYzzwSS2(Zv~sQD_gGtI~fVJ!NPa#B+bs{aOl za@iX(OmMg;mDFUhlSt49ndBS39t^@1F6$LyQtufuv#?kc!20h@@uTN-RzEJRE=#tyY-`DPWNwd{E8^KKWlv! z|L_iZ$)TLKzd96oSDu>N)(_#2ZMw`={*v_5?7{OII1nA$X2KJcOv#r%@$vp=!t`G$ z)c^K3qb`E}X>?G!N*FF~+{?DtCy-|)`E6@d^c~(mT^n^@+DDN~ZoAoq&4tz7K67GE zt%byev$yD@9+4z!P7OOnISk*~6MmR}t*6qFE!)a@<701%FK$($x)^cx9atJp3p)tV z|M&;8e7LErYf=kJ#ee-iCm5IKNLd(W-=OtofVVedXhqL`@!e>~1g+aPQT|-J718AD zk96bGhh(4Zz5Zh>&n}+yuKtrb@Vrk6(JQ`7#_f69&AdID*ae~^q& zH&t_MtY$^ra^sG@&&2zeLnd0?RR&bYs9iQ4n0}ii6h7{&y&?X|ct zmgBBVR}ANTsJLx8;>gD_lb!IQF?-=Qu~5YKc(eKy-7nu#shY=`BxP(Yl!r;59BH2zJuy4r9gq{~lO^MRV{9`~x0|!RE}hL$ru^&Dn&4l- zo%xF_?q<^5WX4H^$!pY`?1{|S2>#Ana;S;D4=t8{~x!X$b1f@J+H z_*~Fz=PfsFM_Ibpui?I+%R;kNx$lhE$%VKF)}8X!GM+4+`+l}5yUW^pG0XIHSA}Gl zKR*5P#%O#|^q8Zgs=h^y&1y@hWS-^wYB`aT7whzeTh;5ycQKNy2p+8P;gGhF7G1d% zsiiM{ak_VQ+)IvwQJiNZ@+Does->!wP*ii-P&$E{Mo4be8nKsvA{TmuRlgq zCO%%aw6#+IlH~An-;?O<($*pErUUY+n(Pi&?~J(={{9{>cya7UaHDd-`*oa6_DwsR zc9%|=FdR8Ol9}bo@g=Tf`NShBW&<=p}bj_gQjZ+PzAQ5M1$= zeNcBv$fc_!%q6bEZnW`T)k}^`4@U2Q+Bbb6O{Syr5k<%&CIubYi`L)N!quO=e=cto z?(UVuR^dIpY5SEjzxT?CUbgyH$<-fPq6U=iv@Q>fyZV;M$O+8tEbIBSKR^1}@efuF z4l~t1mhtk%nUGAc^Xz|?=ji7Rxymd?-s(@tri836ZT?03uWmQdKB(KR>WpGz+NyIhd4<4@OG3B6a~hI4>V{{@_ftC`zTotK=;(-#ig^<%Mp^SGTGi^K%pu+azqh zw)h?XP1!d-xKcSshn`hH|FJPGk+$HC0J(!CtCt1blNZ&T1(ma&ALSsWROoix!}5walafTKxZ%_LWgluF?J|CWstGk>)5V zNREKCB1lRjND4!zG?IFhR6sySSd>^U?r;HjA%vfc!NU5G{z<+?eb7UK$ zt)u|6gqNVd1dX$@y&?CDy%%ZenGq>5D)54cdY-x>Wkc^9|2KAk4xE*c*>#44rCr?c%tP{?_RBUtOUT!R|ivK zF#Ee__3;-8_khJ)_MQz}K{Y<+dlo$~Uyw`&-j6Ww8VsitITkgrHBc}zsw}i=l8}-( z|tPmcTdrCDSH#?K#fLGNnqp7pZ_Etdioimo?8pX zHV%q_3tQo`_>{wxfg$({R3hNK0aa3cW1~L?04cROQ@allqOeL+DLJ^gfsPLVrU+sK z%7M~T>r#_`Bzx6aAYik{Ww_iLAs`q&@NE3qt7&Akj!Vg>#m<7*p?IV{GAu6t>3qs> z^Voq{W7R@)w)=Ze_ogzYKO{seD(hZ`=hJkwl@<2Ro?Xp?MnmLD_K_*G`TMB&^jwQP z!3@Xlrn2&iu2vK^h)^rYVWa_ge#=8!OBR|%_+aM%JiZAZ479}00hj=5R!8~U*)s2W z{@ziriOay4tF(_(j6tK{k|+KrYf;&%i$aW0BeECBLRlG z$J*6(R6PnH83JT~^P0>!a3`QWj)&3AnPci2I^`${mx#s0ZIj_0ZO4PY6fz}8iaA=( zzcHW?ZMAFLqI=PnuAckV#v~AapkG-&^2G3?ku)^1@>k@Uk0a!k(siWV-+b7#x;l7^ zi(vafJ^!MJ;U++Rgv1V*C1{dX|J^I3=H%Vn1VOC~j6Pw$DjDs#(S_AGozI~$hUvf^Kf&p3UkE0k-Qotd81<{(BE5ZoHAz%)U78IAgF~ z>Z3P9I8x!tEzgEQvqzW6Yt zD0V;iH&wP-Q)E^@H0>}p#IM?ov&)cBuql}W;#*cRQe)5}{y|8Su>D5?OWeek;$7c& zvKJ2#t0IzpaW^unf(#}(XkD*_2Q|^tbeBCuLKSaqCSw~oY%{Mpp0NS)sSdX@g9YKq zfxX_JyLDPOKl$cr9*>$yvIQSR)~KeocYgn{Cg@ZRhFf6MDHJ zH@mpQbyI>xq`LY@ymW8%_cWm#qf{ZUb_8D)C*drDZ$eEby?3A6@syNbl8(x;&D8E##;`- zfA#*Vt$nGC6yclgp{r0@s9!3z=-vsgft(?&3Nk_s56_#*GIitWZ}Z|BVcO_u<**TA zVf9au*bRQcupw8SteY(&R5)S^15B$OU(6=$ycXr#G6?3_V>>#(IUCs_VLW&K;R~y7 z#U6(zMx(~TvO$CTZME*_`SCGU`tQnvGGo}l>8O{!pa2h%hw0hYOMkB}-iMAnw`AC^ zxYY6a-UI$fjuc+|l;Wh?!qd$P?KxPThU&LLL7xl)-Sk%*13_JK!;x@CK$+*Od9*Ud zAtc3cg1s>EeL{50;qQ%slg8uo)|EMfU8S<0HKrz}>uCc=K1Db%5r{AV89DNpptG2W z5OD3&^i-aII*nKUGYT)4E3a5p=ZaA}I)DgnZE-qqP`+(*G-Glp8y!e(YA^N{lL=OO1|s*@!T@N#)$HcUCM0Z8HgUyaMeFH6eqx>c3Fh5`B7H03z1 zDeekmcx_`#y80H`Leyo0kCuwN^RH9xecz{oewgq(oA)sVp{w-chuZ@V%reZzL!Hrt zPEx#IPuq8^N6W2kR0cP2k1#Ng{5VQp=^kVJ!7k~uOM&*;MhgoThqq~v5GnZj(t;K%cMz7oj z$r)hk1kbMJT>|w8!oF@`-AIgZ%PT7#zz)Bd?4*s=`Tb6YVy9bvH*4jJ`YVZsqJ}wR!s7=|P8pFMh2%Tfn>7_8IJ8cpUmG!`wUlvhdZXAz!w#Zf#+@pm z?mq|kv$C(>he%o#en-5tQohoFm=D5+HE%qW*kNn}U;kf11PAA9c4lhYdqk8elyYtipW#5fS{FrL<}#*Uj+bV5Yr68G zx*-6Q`a5flW~%Yii`Tp;{^^dNlEOvDXEFs-Ci0mau523oh0$ijsG9v;P5)sfTEg{w z=d!kwrjr$Ig7Vt_tGC;+g;kgNxaD_DmknBy$E#-x9~kP0=%_;##U0_w zshx|lTMlap6M@dO7OnVJE5uha;oHK{xn)S})i3*3QsX6QytKXGSJX8w_Y{Kb>v+gd zul{Ide7@oOWyhwP55=F3@5H*ll?c*EPuE=!!s8j=sC}b*A^3@{pO>id&<`rSq;5y_}N?>w_`jnh_#N1@w^zOt{Y}q$Gcu7)}6|gmrWpqcE znRkqW4}HRcU}kr$auxM1YYMZs6mj^u#KU9e2czfs;IW`I)CtlcW&5q9^AI)+L&;Uo z^nz;pb&*5EHDuSc3_SAB`VI|twtNamLI;IXpT?Nm0(FaX0vUr;k51o@f=C}kLcj=_ zD~877?lZ?8li|_POWqXhM~xm+XxUw#z7?uCDy@oRE)I>TWoM)Y@{>#2RFglgDVs%c zTFZx_1?Izu`-giT6CIdtE879n_VTtPl?HOd8?7sqeiqwB17=~`*GON6z9qva+sm$2 zJWKZ2xCqVf#fuj{ch6yWVZOOT&-GNoRCw5ES$QX>vK#sB_-cG0D4{OA1SjBXo6smh zJqH;#1j#dKB0qh$czAb<@sMxsXy1$N)F*uBcLD-AX%j}=Y30HOKqmrrMS~G%T$}{&2mE2S7FR!H({J zVRyDyMrR4PTSwBI>}B759>9yCK-TBH(u%u$HKOij~+m~D{)NrWN0)b2b5)j9+nDZ}=D`%+>ru!=mE7{RmO|BvFJGSi+cCx>?A_=1GCaRfjx;8DQgara78fuHuTlZ6_;aUVd1hTRkZdLp{(rWJE7tuZ~~`R!ML zUATDFBrJQ#6ydNo);vSM>66aC7*)tsfTAe`#;pTG#fu z6~(a-t2w2%cx|Apf7Jq;G9CRd%vIEyr)act?do6u$fgX9ew06~1P;cpXMSVv-BDtMYMjTKl7xy844YD5GRH3(4lTT%o~gV*AYdfG3##J~j$lO0L->ziV;Y7fm1Q z_dSwEyCQr2kfm+>!{T*X5A5k-W5{iDt8cZc50TbE7*{dKbSkB%L-GSjkLlHaQz zqRV9!hcvOOPxkj>=jOuE>{pFJGe*gQlhQ}rPJcClgivDmD} zOYSdyw6&!jc(UM&lF=QWYwZPlwcoV&wq{H?Fi^R1p2S;|$aL5JE#0EgA=g-EwI~+e7&7*1UFLSwU2H*|h z+#DG8!C6z}GOrHV4X0|s&B5NvGu0Y~@C+^j=DR|cf6_MdyB$1mrFu+1I6GKW{) zh%7f0MZw<4QsI0&LLG8BjCER<#2^{3ZXdX~G7FKKF$btQ0n`nEbYMMXv|hj^9Tsb( z@v9a2u6u@%Dh{1^WzN-pBPPH*DRC-2YwT|AT$R+TNd*R)sttuH#~}*&^($JEBf>|Q z!rwYCQTJX{ zw94X-qFxM8U6_}p=Q}Mt8GSvD&MW7M)^*A}blh;QL>^7Qj-hyQ=jux`4Z-)mOi9I+ z?N_~6DPOjA3xaQ2{LpagM?Wr>_(bA9fIwtoG`v7b#fy(zVoyr61KtMeR{M!J12jiv zuI@~W=9I(9D~~V{^qf^4Dn}B~vtjoTv?r*D&0Uuek`PseYsz+FjoG1@bNja1O?POW z`*e;ApusJ*+dIX(^xD{z#bWJ2COPk2{sve)5)?kmE3}#qhdrmPt*n|s!A}LsPsjpx z$1S!4BnJkp^_CAtme6~_UO+A|qL=jM3uicC!Cgugik(QE;7@u}gV@IcTSLT`iDdu9 zgLQrBhDIj^}A2dsk20H#ada9}2=P#IiU3vZIgm+qcNf0Yl47aquMVt3=G-i~U(?{Pcuy^c^TvL8kUuJ)5K8 z0%#Dh-?S8UHy!HA3Nc#FX{DUExs9O@hti+HUU&iS@V)QPnGZ3~E5t|1kO-vUvCWrd zZwnO*bNaZmwc{srHTlHOHcTtH68#$b1Fq(-=1WHWbWy<}Q62erN?~1ftS_>mKU|A= zBjZCV61E_B^7M#Loi=Lz(u4V-tL@8TEikrJAekg+9(a0-V4g7kXle{N< zL(!H1T&gk2Icd9IOLr;ZS+``KL{s~X;(MB=xFH$=r5hR-H{jEI0vcMXdzOFuWb7D^ zc@gK{6H$9gKFF7y-&Y{l5OJY~c85Na?Xfv1w%kSRBE_OX{v$F0bC$Qzu_z4HYBoKs zB4Uc8H^p5bUXxt&1o9hAdw*#uCEksIix|Mg5D=m3t0i%#RD_jAwC$lv!R8D6uM0zz zO*2*r)O@yQjJ($x4LYJybO2<11jy+0a^aTB$m{C3uiJHxxO!;>U=eRW##8SN$Soxd zN~(H==iD&#I0Z`mWWTi+6CDj3xSUyzYF$knqmP@CwP&WB-X}9mr6^$<2kqM!Aa!+} ztIM1fEYK1MSx@!juAFF|1Ulsd7PVnK6-Yj<>pBeKM;A6OO~$EptjZMfP*r%a)62Sh z&u!!~T;E@c?U_Gz-ue08390Mvfev-~=PcDFe(`8{?XTn(?~LLX0K+FM|dC3W9TG36Eo=J z*LTw}Ycxy#gZlX^tI}MbKeEH5JJnD@0i*G`%yY_B5|gfmzI$e1g=9#x`j@&wcpvZ0 zLKeWH=q0r<2&bA8eM7?ui4GkSC+|REW1PjG);1>S!q80(tbKL0s$q|9vs!cF<$I6k zW>nQD)aN?0i4hrX1nX+l$n#2qI1R8pat;<2e^A*JnGe%QsRWwLngsT#^5{wq(DW4) zn2i>5FbEUg0klLM?Ad3>$}@O8;>BR`8YJ{{8`8%dx+k!f91TYkW9y zxyz@32qb{nF~Fq|!0ZC?%6kB2hop{8%>R>+LR(;iGq~EC-8R8eV_qi$cg@bSL$FJ=G4iC^iQM6r4alRfnpG0|bL9jCpLC1$WWqN4E%P5x3Ks!+m0 z7-x26Zw1Ft$R9FAOtm!Qb|=sI@A=2tw63jhr)mT$|8H4Vj#h5`u7b$Fq-jw60=S02 zogM&^TxSSa^F^R zuG4stHDxq;)_ggyt*@yBtXW)eoGG*E`?$@%6!?zS{6o^m$Ah7F`Hx#WdmU}Zg5W3; z`vQ9i+QdRb4MIK;m)_4W=;e$j>}^0Elfa+?eo(NigZ~Ob3sm&==`IFe)DN$0Pm%Zg zkoZwQBdjQxk4BFSu+iA33C1SKf8tQoGv8-`#rw>!W^8Km1_;VWO^>qLdb0$qp*42l z^a<^Gl%>>Ca7HU3mrpZlc$rB^ytH*xF+_p==WU`-?C7A>!&?CD0H$-hl`HMj*?K$6kEj{Xb86_yfm#>bme8U4L$(w@BT%_GrD}NvYpbVVx&c1WG{Zt- zQ@bn9sui+G>gutlL(9*>0V7Lxc;Imx+uJY03y`_G`nkKfKvR7dQgZ`oEXWA3CR{a+ zOmMNlF#@3MzRk`~h-oLJnL?sfOJYJq!~-zK$00!m3r@}XV|x$@LP`#iHHrp&%K#Uu z=Js0N<+oUs_`}m(Pz$upx!xiH8v#^?=|77Ee)xaBIh1t5B +

+ + + + + + + + + + + +
values
Insertion order
values[0]
values[1]
...
object
weakrefs
dict pointer
GC info 0
GC info 1
refcount
__class__
values flags
values[0]
values[1]
...
Insertion order
> + + ] + + class [ + shape = none + label = < + + + + + +
class
...
dict_offset
...
cached_keys
> + ] + + keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"] + NULL [ label = " NULL"; shape="plain"] + object:w -> NULL + object:h -> class:head + object:dv -> NULL + class:k -> keys + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_313.png b/Objects/object_layout_313.png new file mode 100644 index 0000000000000000000000000000000000000000..f7059c286c84e6cd9a71f59b640a628a110d80ac GIT binary patch literal 35055 zcmb@ucRbgB-#7eip&_JXOG6q+nVBJFWu#?hq--*iRfv*Q(m+UNGP7rPsTA4DOqrqV zc|YI3<2cXjy6@{e&wsAFqvJRpzwhrmKI8pP1nmaa4HOE6R_UaI7KO5U zfkIhjLPLdDUg}jY;s2*&%ixWa9lpPc$g=5;T5hLHP>78lnlo|hQByH!t z%jYAnarCpBYxY0B*~|QN;0%KUukh6T?ifu4i}^-AzQO&W&)TiR0`~OFp8fgg*o(OS zx7{BTy>EOx{JM*4xiz@P;ptXM*V!V&w7&%-*)8MWC4P9$NL!}yG0M@faZ@o7ip^qOMHzsNxI&!4qF3S-OUEPii_2ekBH{q~8+0Bj{ zEsj^v@g6u(@t9B7uBY6O`ml|eS$u6SeS&(F=Z?ND3=H)ZLGPtCk8EN&ymr;9RfmLy zt0o7(_Wb^xI(i>J7%+4xdXy5t$+N+4wan6FNKVC$ojZ40Z0T&<`&>&~+qcEwI_=W@ zluumTo=vp0Mwc)9*S|brvM@bLoAbTYONMF_Gv)N@(@&p0d*QXRJUjB{bL>)-$Mmyy z`QnL*iTxg?moJ;Vf4FC$vsgk^P0h&Ic(uAZxq3TfWDKo84!5N3I(&GG)b(lKXU|x+ zZdk|ZF{$Z$>sD`vZE01CZZ7W1s_omihdPsYsPgy}7cUh@9mMf9c-%aBA~qpmBmV6w z*{9**fl*$|7q&7n-DxrO(x=cjY9sHhbEsXd^vKa}FFU+CPkG}G16y4?T7 z@K#6&<58y}-|dF(n)`$A-BYx%uz1<`IZ}#k-MV!Rvz(lqb`B2mCr{E)x`&#QU3>ht zYY!QgdU8_3@4i0B>`mg5K=G?x0d(z?5 zse9GZxbMT(QBR(zp1&ow;@iY!THLS#A@l^Rt~>DAAYuJ*NJ;p zcJ5d|uGg=tbc7kW2|EsG>{Gf6xwKcWR#Rr@ z=8QUvuF+O2;;cuvhj=uwaCt3eWGVF5J(sW__`?0I=o(}4*%Ub=BW}vY#o3A3sFgXc zpwV+_20^c1AJi*$+csA8_ntN<*0#F(XT_SXqH9ME9y~~)7r-U585qTXSu>I=xmru*!EHII!`}<+TzJS4MBE(r{;StkqQ6w|{?mfz$B#cb6`1DJhQr{{De4FV?ut{0_Pt zd5!9Hyy^{A9(j}_V`F3OcF)-!|Jm8T;E9$P^*ouazCFPe9S9yph7_KB6$p?B}zc}GTaj*N~9;^H4PC}J*lo6;IQARs`uvNYzE zVcR9GmuI)Csj11%(XldsVOP52pjKW%fwB|!iwlmo`?Ji7!i5VgBO@b+oQAZqYi@mY zCpUE$DR+$;|5V<_$9AlAnn#)%8&Arxv9Op6ZXv7ZmSEdni%NekPZ};RuC3d*_kDRG zpO~6TMd1xqo*e5k{7{dLz|a&tJi;g(gPOp*Z{LmQ&pBSbdeuEJ5K!hrZ8G<#KRqL( z-r1F0xvAa=4}9&;og7u>lH>)2|DplRdh;jEjd`sER#qxl`yU&X)zgc}kw*b-FfV$e zZn1@t@xh}0LGyYZgynV9|PUhD{owT`-UAJtfQ{34%CN-OVeSNP;Y{HK{ zuaWqlx{dH@M@J#k!z}n4@%(u<(+Ml9gGI%~+)qBlxt@xWPRVw~wMrZ{%2aPu#mUmk zz5Vb(#Zr)6!g;z208Tl3dwY7hi}f+bHF&q; zFYe+O=kqw@#_I7p>3aD@{*2t5oaU>$$xDxMncZk`?XO6z#R0p~9BNAtIUpu>-fdi) zC#2{0-pKpY9Ab79?65+YvB$OL6%}WO&Ye5AhJv4_XJWdAT~hJx9St`0(?dq9Pen>5 z9X4=s5;it55j3u(Y3=Bc=YR6@@@GwmJp?}O=KcdW|&9Z5+^PbIG% zew~sMm$|sO$Wz0@#YOi_^_Fev5)&oX-+_8FI}HU3v@$uZCMI_cA$;NTkUvsDz`T$?I^(#4&l3x7Tc#Qixs+?-O8s$WP?Pfy`q zyJ^#=`ynA~u&72_GuCjqkFT=%mUlzme{)ZD5Zn0F)Ny0un8xLWvC^7H`%WLSYCRMo z;j)>9h2^P)3;*MT7jEKeD)!OR(pEj(qjCsm_$|+VjU3I#C>bs}n!9YGn<+f9kaes3-DkT~k{`>x-{?Y-!wS{`0r zUJA#LD|Kw#&R;oJx^nO7Q|5z)CG1R0Osst0?^5g*>Q=&pElEKVZa>RL8tg`vjT<+H{*6yd zOS@V!H92|F+oYb#e*P@)?Bwk7 zT;Ky7@z#8)XoRz${TXh1$l<5zpv^2!lsh{(IM^Sx>E_Lw+Vk1Dxj02ByWYz6rerPYE55-N>YU~)!o;(#>2xSHa=dY^^`6)zTmY#Kd#%@nwqXXa^wgAN=0icGp-fj zm!BTLzgCj3R+57MW{%h>Ve12-%FJwRx!s!Bve_JDWrhoyQ1$lpWmtEJ9K4XZRyW70 z4Cs$6E^NVzSe{SC9e7IShYqY=x9->r4N9JrS8-w@4FwnMg6+!S;Gmtev!bRZL+7<$ z93$=78=pOU#(VIfcG9`CXXBfv0hNx~*&RX!d2s)}Hx~YuukmWhE(dX-*_l79+wM;d zw^X#WY{&b=SFc`8q4-eKRTo|Rd&qNfma4;|B33D6hcE7*UT0EfrdEFhcK6G?JmyQ6 zE|F~~v$D9WX&*no;-yP_Y&wf>|M_$EQE2Fk^mO{|0tV#Y`jnT;4Za*~TjwWt;~s}Z zj7iPC^z3ZEprD}koafJYKy?3L-e@W|14(psC%)oyGOp|GhWSS zM+2{KO{^!TYMs!`G$>({S8pJBZ2 z>eZ`cfrRFeW7`|$eiPTV?DJ>!OI^CQ#eTS@eK_I!?@h&=N0`x|8JL*jwbF`;iaJ|A ze)`n&CcTCnsBmbfEd}5*6JJaGdA4bQBxgezaY9&}(J3k8aSo=akcS z3O_%;os*Maa`FK^Jw056yQiKXEh{Tq<>tm$dP+`Nc|&h+@Ay=Up-J=W3+MB%#!X&V z2A0_v;^D?i&X`Ta>g~@@;r8R*$C4z)#F&5n{MiZ+g__f}CE-l+*`D#9il@SsOav`$ zVilqh78VW&46LwBE54;Jv!tyYy|lD6@%OWg$)_-3*Xg#)MrLNWu&zjdQM;3Y9o?Gm zuvhUe^Nsb4Jl%j1Y9((vu;q_os9X@>64oK2{X{zPjr%$2rTUc0G%j@eIQd3h08sY-mYdSlZ zBYV}<)cjt&;06{MN1g9`^I5tAHD!E$IPE3w;!NXS@Q4Ugxv$^WTD0(}g{`IFbiQ8Q z{R+4b%jx!qCj!Z@%m09U^^%W1J92q^;AQZu>_YoKs!rEF+RB<5An!HGaa)S7&r}Br zlmzrt_^V9)`TIA+s_pPdM=o7{etvppX8BV=6RgDGrew71`RP#*QX1@;S9ddN3wHtn9I-I8zMcgV=NJ`l zvsDF&Bi-c33i(RAo|}^n-&I;CCnvjq{D_^9yNEWfRb0rz%q;stqZ^D%(bd&8q{O54 z^JnbJl&sOw(ICLNr%#^>ou|w5xWNv10hW;M=`z~xM*#2jw#-YeeLOw06uy`T(^JnwkoegJHrwl~8bqLt6@amnyYxl7x27P%$l?b)-3o0s?G=uN`3X=rJWo0`T>T0i4i zUS57GV$BNL^-|08&kv=^-{09#3*=F*UQUSQN;5p*C%84nAL!|}BlU_6F7wHgC*Ph{ zmuk8qgMC1*`QqGP21Z67Vaw)|0*;me85x3L-Xb>NXk=t$viDg7)2YRv_ZsyDO0Nwo3(sJU=nQh)30`|k-3q^6v-J_$l0s;cAZf+t+s}nU} zreir4O}AS<6*AxAxiBi4JqI-0C~~L`my~G;wf z^CxdFExj%-HLlvQd(WO8uzr+@37kHlg|l&4sdmLc0vYF zOfx8s*5K7~WS5eXBKXH+ezFY2axKt~Nc0i)}(~*#{tSljD6P9}0!xQ(Q_nAwqeLfYvW$EY|>dguO z@JxI(`eJeodHI{QJ2@ zH_c{${ikqIwvn{-^!v86dv;oI6NZ5TfGt+`#6mMU>3R}VcF{S5YwNJjJr<^qca?a& z(B9AWE|1G&k^wmTHn3f6XXrJ2hC+dqk(3)%TdN!))`R8PErIJEpC-6SQgU#aC zrw^i-3cAnQISzbT_57%l90Ui{Vjm!Zn8d`yGu3B!x6YzGk;+LYJu7R%fx6&0+)S&a zq;yhUz2*GnntL4R>QyGeQqJo3%*@O?_Uwsk8e(LSvVtBGaMIs#w0)bRqGH3*g@0Gd zlDVI3+Jq+n*$ZBLOPQUWjkfJ7VHUbVk5|U7k8n!4UBp_)`qwFTYEIP;Oo(*bU{Zaz z3}O*2?$>!8o!<6r%YfkE6WZEarI%;>sFz1pF+4~mdR?six9{IIb#;@qczo|~@t}|n zy}YzA(HATe^XkqydG`zt2m1Q1Mdvx0@?7qPthxCxgHD7D6gw25TP_RHDB}tz#BjeI-@i)3{A}^ER1zvaPUP=4im>w$G0D! z9ckL)(LKE-TrPm8X1t7=t5G79nle8*=)NeRDbZ@lLA?1+J=--S|OT+=435ctQe*J^^W_eD3eB@J0h{ zY&ja#$L{PFgNn-AjMrwu15Q#yq7+f1xzXOPlWo5K(+dq=?Bf(??iYM~_Nq?p?HQUM zus@GEJBy_1<=-UDY4@a)%#$Zi*5p0Uk`NP%Mu#0}OkxCo^3Tf3a%;&I4PxA~%Rkz3LXPXMA3l?=EI!sRXr2{s^S^lj3~A#DKw?#X_WmF4?yjqdmNLKDy# zKpFCTU-BI;i;0N|UB48Da*(upVSavopfzLH*;KvT7v7kaz4i8*za7xNG1Xd;P7fI8 zrjL&i2o6y9CQixoeL|0BXIA8fKh=n$Q@XM&v1QAaakNw8(e`ZDiK^{_;9ejtR0akH zc6N5&_%dj6Ur>GTa!Rp&mYQJ)e)4AAdrH>OkV{%x`h1=pxAfc(M)sqQG^q5ve0;|t zpn-__pq3d`+@f`b3?K+iND>>{?&{Toggv}cl9JI03DhXvt!-^aP^k&(_*wV-gF;k5 zVljh)%QG?i7$~mB$&=3`Bl#aG$KiupWERJLc=`F6yq2b(KoW@^_?G|Xkch}x%Of`5 z#8$~Zdh$f&lDN2d`KM26P$L4r7rF!j*x&i~O>Y7fZzF}Apq4&v+XNIcQHir?U?_t; z?j9ahX{F2im6eqhu3pU?Xi9!^2--HTx>jIKUEQgX&SG6Cd$$`KwaF#r4tZY*49g}J zFY%!!h;47s97@rKd_8`&#YSVRfmWZs)#zN&_f7K=fu&rMD`ptM5=x{7Q(ASA^gouUF)%xd>?43pCpc8G#X$ zk^$=bcH5Gzzl9nJsF zUR!qzNRl}=rI+#HQ3K@U5<6fEq?ui9ay~p z&3@0H>xWc7KXQ39Zp(dV%}Ns)c&F|G)uFJ-xd_Ll)@WGFs2kW{D_VRaM1WBP=3v3kQ3UkB@AAOG`^^ zrwXWa^ZCC`Oib3Wv9aL=`4<{l?fF-fxVgEBj06Qen_~k5gEscKOjt&Bu?b%9F#BBW;ZL?%fk}8rq5$)!p5_ zql%H4IZ>+xiUbTHLaYhzgc2lVt_PVb)EcFfA`59fBxDIS2|Y+@JC_qUBb*<)2r>KB zD2fmm&bD=xdadwGb#3{V++j)qP1L@yO%WR!bg&@@H3`K9yaw>{$5nAc2L=Xc)~s1W zKn0K`ly(2xx7R~FiuEl4gKqdt1QJ2PYZq%0X~90*H{ZbWq2XDWeB1clPgqc}-qk+~ zw^mJg`T(x5w)R>mLy%T-6%)6_X(eS8;omjLKZRI8TYcI^R7O-Z_kABQ=_{+2l$?gf z{-eWn58_LHKZ=Y@eLNEW?3u85-GZ=rL#$cx>)Q~|!u;t58rGPg-lBeLYinO|FYFIu z6Ak+o426CF{{2JGD zcvu)&Obt2EAw;a(8)NP9>o89WySwxelyBQ)zkuPy^7`)YNnbHUUv{y32gRh3?_< z7+mY$wws41CM#=eO)ytYmigC)nB@jieHTJ!988C)L~Ryc9OfkhEIvwlLcpLb%4#iuw?Kl+`* z!e`b}Q$sA=Xs+}g-e7>YH^uyR1(;?a;7k&q$1H%F$`Z7f!0~aro}WK?;9dIp`1m|M zeCZBMlG7{#d-fbBZ5PJndMx)0R2H79!T$a|5fKrU_?{~4FJG|Y`ycwtA)+io7`B)i zyH|s*dzV#MnO)3I_SNYw!96@YuR;Y-4QaR=YjM}Z-L25Njp2C4s662>|MEqnwY~k$ zgIy=?z$4Op_0oK#dHL&j^yALmASGj_a^r(yT3CX+Da1K=ysMty*4CCNMnE+kF9st1 z#g~2Qe!<4gK76W=8&_gygC-#}_~W=W#3QB~jG4kPeWhVP;*GjHt6+S=MIj@9_8 z!nglvOt?%d1^N3^gGRJ3u{kD9H5MOxf}_#dKK4I3=5&4`d)1A9f6Iq@;w$&U8-MX7 ziuc%WgOv5b_VJ~cGwJy8!&LI`$0IN%A#cT4oKXLdEP#J#=r+#lBZ7;=hAZW_XxaX; zy4{6j|I+|LKiv`X-VSh>X?W$w`w*HuDz&4Hb3@a`apDi@a*=d=J1 z{DXt(01+-)S_bBJx~Z6S3=CYcYR_5^L=~ZJ1ACrfm*U0NbR*(-;BZ-ky9t96B|jYK8YPxBv78lPYD>by`x~1jt!tY@l^6e0*4%zoA?5n4?;j+APg%|id9qI@Z7gcl=lGp;*OP?K zjWsDEq~K-q5h?xX)gLrO*B_4Tm>y~xF6je5e8L$M`rZ^g_RD$Cxj&cSzR4zOm$$dG z!Ufn*x6<6SGTmXjO4d!xrTgs-t>-syt|lt{@87=x=>B&Yt&|ZT8%y!7zDz;Ypis=s z&1al<;_wGvRG3D!zlJbP@eU0=^QOb>^5r#Go+TZI#tEYhNQw5bzrX)36k#~(0YO1k zSpRpweEH%w43>l&23ARtwXq>sFdZHMpce8HT%}9XY=8nl_};LYp4}lA^Gv26xtnn%h@Oir3Rp!&O-=3V=hp*vJ;QGJea`*h-7 z3=HnUOc8B`7qXV}=<(xMqsa%7ljU*r1;ONd;D zmsrOQua3PPb;yEdke8R2sK!W2_V+5$0j=<9q7y^Jz;TiQz|D0VzZ7400hJ`$4oui; z0RD|Ir?RvApYZ7t;z-;_tX86t6OXT`NYZ^}X$HGi+-+*e4-Uk6l@kdTO|Q;~xz8R# z11A(83M>{hEI@A9894e!@qZgnKp#M36GWng@B$`af1+bQ=GVV7Iq3*Wabk3xDeh=& z#oBs(o>LLL_;2FqA#H$2xsPGODtU#48&T6h(tW` z>*I83;5ry#KTvIE=LK|7fYtQUWUM_~g`zP>WIBLy%@p^+hB!Hz7iT#EZZo(jM*Ag5J@^gFk)zJfT(`|{5;B8Sknp1J3&F)7XM-~ zHqM*M2{{}V8Z0Wvt*fr8G8f;{a7My;L>8**o2V)@)|j6?|CtmpB>|LxlPBPjliTzE z>eS=~^w|I7E$%-2NlGdz@@{TNVTNRJY#2<;%sg!EkA2+8PxLtad2p}yxrbI?PtPhk zI=Yf$PjuA2vpE6-Bsw8U@UJ)gSD~~?4Y6x~c7qUaMCgil>jwOVmUHpPy9cC#FE9R8 zr{NaLQ9E_28vg7WTqCSIfWw}Q8qSK+BM)1r@ca z7n~mLpvDm~GmqD!YsE}Mc*-_M$qu-CR{`mxi&j>5TU*cnfTVEb2qSowUtr+In3$L^ z3F=&A38ah?F9NRH3-o7PP+7wGpG)||I_m{DCXtf@=g~VuLqk-35Vy*91xwzHj*ecC zj>r4C!A@Zb5Lc}(N`~(8?C3Xl8;}a7%;Kp^j4f6$AzkK;J!~83cFuXoh|(Nc4-g_yhBLj|cX`>v(t3)n$aw5+~h)l^VW6tTwwr_`_k=(ayg zwEev%TQ`J9Z8arDp!nMFZ{m{EttJBi{ZRf5Y>VNFLOtq+nLp5y#zlc%x=sonbC$KS z@lNucFo&o`zV{oh%x@=BDP)R@%F5WP1;`Knq-~2yNWgW=t+(ap%qoZzo%`kg@AZ*= zAzZ{-&e}S~aYB|)v_Glt5LhUb0|%1LazEA8QBlBoyIk1+|7?#yV1C$( z?b}Hw6c#D2?%uXBUMp~LY^wOOYp2?&H}14P1x9kvTLGZIwqK7Y2=S|G|mf#Z6( z#PS*%^dP9j$FlCoAs!{Fg`PgHZ9teGIgwMRPEj;$Gu>gv&~Ouymh-v3KBFurrL3mr z#HEg^s&#KyJU)DR3SP2`vU<&$Gqo_fNehQBKGIcMx^Cwla#3!Q@A|)Kd01o$YN#&l zuYW~e-wd|}0gmf$&V76ahXN){Ixs8MaPee=%9pQS<&~9dmwes2pgTi^fcCNhYdLu| zExg3U)^;UYeix3&pBeVmn*x`yd$2uNU>6XXUEfwpP33|cYDm?cZDO8_VnneE8ig>J zVc(~y8Y#(w;HqUt-dGbNBdaKAhG+JrArpXlyW`0RL5oISYhMzZquZDIM%}s{9sS;` zSNlm%0qxkxbWnEY_jdq!J0z4Km$8kNEbvcGO~q}uXu6PN#iFdDA_xmMHZINxUIb*j zBUsAgFrr9u3wqmj>E&4yK91+F{%rx%(9!vWW<*7CLHD5Fw#^Uy%j#R6W>_6T8I4nWnG$L!C$2 zwr@}9KY8-pC=y9HY`VqKDxA?M?f=NY@Q(j6v)s$(sHCQ&!`krmD>zWhsK&_?$QQ!A z!fJ57K^#5&aojpK62gj_k47?1ThM4R)ETXUFHQ;k&o_kpM{e#vTAlfxHi7@F-aD)} zsmJaF-?UgTEA@G!9tKYXDlfx=|$Rh8{Fuf;^G%3hniHF zSy*mEs#9`uxO&w$JUl%8%^OBQOs3<j%|c6vtW60(oeS!RZx>Qc zz&se{b~IfJ!LVn~Zr5;m>6t9HwY9}S7Ahwi`S-$eC`Ew?+-!syjkx$ua)?1D3v+X~ z;XU5LE_zLm~uRh%dQiDhhD{-|qIT)1ZUi47bj#Ck0nyL|E6RS>qODCcb zy9yX8xa30w9Uj&WEwkyDdg^0~Yrr*9VrFIC0O?f_NPGfZvbn?X`(=o}TF)UCXShxn z*CUBwWNNw_JWoMZ)|<#n5PF|gyWP5VOD;+K4zB4x3G`QgfyGr!zRfL!-M_yhR99EG zg{=$1Hi7znQ0ej8TW;LA(F+S+YY7JY6-iDHkEO>;boFvWo-#_JP*#;neUQg&-|Qcd zsBbCMMjicd_j>M1WP|rmXwd7EMxI@cWS~@@GxUhl%8iMU6FYj8*aont;{H@WT!LMZ z{74<=zUM{qSB*>=xG<|^;cTCYlia)a1k{c|><0_>>u`rz7p_0OqC73UXBpRL8?p%W zz$;oEIZsDg%tvmH7zm92UY2@Y=!&%5c5;tJZ9BI@64PoZK^!F1O|AG{%GgXc*!}+Z z%ikxmE30gE%S~5$9FX>zXiyQjUBh`@6+YAkSS9r+mV$628MwFvppM5COzl~~6Okta+CItAA2vgP+!fg_zl4MYqQt@YKvv++`;fhX#CquOvUV8S z!jQf9m4ks07OYQhZm#kLWaWF1=*5yEOa(e}YqsTf%8o;aG~!bK#XZw6ePBZGa z@ru<+iZ`NxwK;&MlpS#VL_|c?hIUCybE1b@sJNq8zeHA0%)WOmLM2US4F}P*bqoP6 zDbG+g}i?+!mmIB6`=LRSOe+pDW2(+M>J5xw)3vI!%v~s&(vJ*-iS{!kiZ-wNhHhR z4)57@s_C5l{G>T(K`)dt0tO+e5gPdD(N=UqBQ$bX>~~^=sD+CVtWH8(SoYy(p1_+k z7iv4VKa}+Ym{b6KTs;ghK(^=IzyBr-0Ys^qhMM;r#adI_{1IFJQU?OVl&~Vb%g7Yo zxpPNo+9?72GTYSftMV2k$3=P6BfrIFblm(YX{YmqHwqSo*Yez6D=RB7sCHJh?WS*w z4ZFXm{O)!7_*Bqh0lHY@PS8xNh{-X{YFpD(a)%INENWfkb-h0qu zY9^40?_tgp0mFN614!~7#|06o25)!}8F>#UOz{SRSX^3qk(NdWOMaP`ir%%@>v|Q4QFWl0dkSu^-Yn~N}1Ps~_F@B&ge~b`k6acPU6EMDfrC>*4)J93Q zrAC*D_C~jvs}FRe7K&&06rB6_;6@G^+&H^eTs&XN;}bLh6m~*DV6%H89gtm`>=ek2 zdLoAeHl{sr%GED}rSBk2xCObG8&OeFwW3y^l2i~B5h|E8!}LZw*B@BFY$)O23^26U z{4?=D@kY+zFRg!;O^gO@k?cZmE~T;%K5l0hQV# z;Rg$a7EK7bs|Q0V|MF3QKWieTt|M=K7q*G&^P}6qdhmw)v1=c~h`x++3+N$>FM;Ag zMi5j4^9u_}=#h|N=vpLcjz!mt^o6{m# zT=nffHWRUN+dWadSfAfyVrK_++k}^x;m(w!sKk}Ffx%Tm`yET-2fS+d^0g4xjQGOD z`z4kLj5wm@K(VSsq&Vg*`uBe$y~G|3dn`%3UlN_ynS%lBm*)|Fq93j#@6?lr4|h`0 zaUKNv^2K1!`BKkfWz~OsDh-(dICW#_7rm&tkA|+W7Ct!r_2>6|R`|bua7t65SEVC+ zVxBcxk{WiHKMA9HyVE5FM&2;jK)D^ z2N(nLiwj*VOmIMZantut4V~@FzxFr2)|iwtDq%Rk@g-E za@r?(;v0t{vU=5_nMeom5x53>j?MSWuYsW&8@iV)T^1Eb+orJjy zV+0Gep2au0V$q*jF8TT6$H^&uef_vU;2!I_eX&&N4?j~iy>uz*ZZ7CnQYzB6Z-&wT zMB#`I_8`*2IRRi2NDMKEEyBsxx~kLR4#%1wZ7iB^Zc|rPeKhpVV`2ynFc5A`rfO2W zTNwY)*@o+rrto0B=E~^=@B)zV6OgqTOLpzh7yMuD7`OhJ8OpUaUY?0T?>~H~ z@LE|gB-vG9d|3>Tuyb&b0TPmqMh1wCFUZNuH?_~t&GkTp5Oess3Fhiuh>9E}m(!ko z!0hAGe>N9}86FS0AUZ_q9hq!E3ns%ur0e4TB11>e3rvFUu9am`K+jdt7n~5}A#-N8 zK-)z~f=GexRyDOF@EuhNX!aju>dPQPugC-=Z$ z4rPJB|7KAB`}1+4w{f-#3Tt7+60m_#!s(74dIn7iOguII-nw<`MVO{oH~#2M(+|O{_+lPd3#yAtn@&GzIWWX|DYCm|^Fkt(TbsL|VsX?4N5CvGI^?w}jIT>02 z14pvcV@AIS^Eo722xE^-!;k?Vphs(GDYseDQu=E{5qQ}sDJ3<5jt6{tqQeO8szf=2 zo6vu&PJ)7h;CnaWTw+3iBw`VmDp2y6U0B!!RTEJ_@1P)BCVstuq|*sV*-!zNKN=z< zSC9+eVx|sC3mMY|yMXN>A6FXR{L=q4a<`NnQc@hii6~Tz(7oaU)o}jJmE@QI5vPDv zU*-sKbFTujpv|EJuIl<}a(PT#JS@(|7`n&cI{p z49ThB?Cu+sOyNVTDe#_S3y@K&A#+NB+XZ&+*g-)+gjvch%dxEfn_B_E)2XBW1%Ku+ zzQHQ>8%jxIV27$n8)%Q38&A-MQA?b1G(9ft#I27?kuWc6Rs)MZb-9LB9#OWowL?MF%h0B+IpsQ4djBlzwJG>4Tk?xQj4ntim$iF=l-KFf)Ct6(L zY8(JT>;M&HAPr~*Q4srLYxu1g<_bmv4itb2Gy=P_28&|?ezk=a8H-rNum&0a!I;Se zyi-`1Kuq6HW7R_q0J#RYnI5_A(s_*2g;2Ke7O%3XwDuFbyn!vO_2&2Y}Y?Pp0FLbwYlif~*?s zkku4&p%Cv_McK?HeHY10quNLNz{BN0Q6`9Z2o6eS31Xjy=>&7~JFWL5{~~H-k(zKb zd;>|EAzx@Nesna?eu|2zR8L`9ampUE985o2)5+*bXO{}eh;94|G=tD_IUV?#j zrmeNL@%nZzm}vVu0F|bii=Vhnjw35tg(NQnE9)JkM=d2HL{9L89P83pDfw_aAkm%_ zPMt59vq4e{Mp^sr1LqFB{&cSNosG@Ee~X?Wdj}6fzKxHG-LRL{syQ$wlVK&w7!kGH5rg%ucktO|MAb#mjqhV|fkVRNt0;N-=wQ z7;NG~&2^&mPG&TIF*H0PtXXl@{A;a9?ze`+GkZRMHU}weebdR4?MVi6XgvImJg50x3@Sm$53%}*xj`?Sc=c7pqFbrohb{Y z3mKCDH|2+TE4>^I_s~A2=_+|^40FR6gM$twFq`94sD6q2GZ+G2^XyH~Pn=p}G%X}VSf40s8Zl7DDvNwVUv3MJx83P%d$6tL z&8mSz(Q7KC+0KUdY~WT%d5Ckgcn>*V3B-tm`t=JO@hh1*@>{09n7jyPc+R30$jP_q zp7|f4k0W9spQ*hJHa_?s?pwNL)@`;WYtxnYAM>6maNe#=eZfKLDw*c9U+8n~OAwtM zul1i~?|Ae0DGnQM>-{@IIpo>s~iok@QRhODY2nUUQgqajQ)Pma=yTi zzSlr0Q$T@lpI6}PpddBA6FWK2m^GWJN*qnesYrat_Fj~d?wI~M{abCB^0{~oUReYE z9loxW?}a!g-X=9!6#FHt=(+3Y>AgP3iSe}qhS?U|;8KwafhdGduFWT4$$fl$#lzxc zzQOVsZr|T zn!;q!;=NPxn!}2e5uL2$_EcROZi}0{7hjzG&Ro=1)r zo%Q(4UvOn{#{qf_Nt?E3nI~$$vIw$8ztKZY0kw!Ek{<=kn9Kwq-3b}KA8rzbsB?Ig z0Mti8btjG=hY1~T5i}UJ|GEq&xbNts{6y@zvFg{aUo~Xp6$;0(!fboK-m;Uy>U)p! zvYu8seaq_z`{z@UrUrGUeOW=Vh7DaQubjG?6)r2ju^KphYWb?x*m-9Qk&V{{ViThS zj%}o_ADJ-wb};WfSJz@0or!1kj%4ph6SoEj)z1r&g$ACf;d7!aixU@%3a^EPgjIY# z;%TPzZVT0Mx|k2?{8ST)`hjxhuMXXPM-8Fle#bXm5L!EC)A{XWQCYi^ z?t&F zwygRbYh7rAu#z0JYjQCs3kic}b2HaEC$!wppM$Y0?>~C9_K_5~m|ZvJrD9Mo(yIl7 z{J%a&dF|NMb0(ilH}`Rn1%j`-X{nf{Krx`QZk)^xpuYTfA$;Gv{W}~_@$#+FF-D;a zPh*p3qN}$)ej&-(%!;S)D%Iyt*EW}SdOp1nZq#Qbuc1JnvRe0y?Z^IKVfxPe4r~q) z#m}z|{(Yh$tTFLbLgqopEWKuJ#JC4r5_>(;`CJue~J0h$K;&M z6tH-tm1GKrfd>#A1|CO!d7o$BH#zj^(Ia}ydScQVg&YCY3%M1F$o7TS{rdS+5o0O_ z$Zc)_&=pK%V`Wvgw?|PEq^cqE7^|p_D*jm`XJGbQerCNQgl*KPxQRmEu zK9y=DvEnRK*!68xum7uyHvdeC^obW5U(7Z3F6SZ>GnkSrJky(*wQf z>Dg~jnpfUzT!%__Zg!vHLA}mYdO68W@`1h+sQ z_y_nuCJb3VhKs7(>X#v}P`B*0n}Lp@tMKZb+Rq0;DybB{Xd21&XHCc<1vl-sSUDk%sx4GTt5Yg-B0eA zmX=oB7_~Qmlh@(|HHK4n9z1v#38_O!OF`F^!^12DMMRnde|e@JZ{+hxn5t^%eCTw# z+1~T>H7rQ2x`gr30b9XrkrYRXvade&R=w}$lZM6LjBX0kqa8}NXjXX5EwLxuTXeOf zlT^FweK)~vPMKi<(eLsiZn&Hqdlh9_>vn3znN={ApDi9e#=JzZH_38hURt6&ziPpQ%VnVwAbJ^x%y|oz_$RlgmNwNFaH4 z{ajXDnOQK2Et)#_bIBkVbY-hwPG(3{)vlNB8$bS=`xMO0&T43Ayak%G6ut*l`N)+o z(aLGXuG#>n|K>ei4s7PzzqU?x^7KwuX=-WT)IA_NU-Z>qN@a$=FxQIp+>;mjKi`X7dv9axwV69JZ35d%_##64&OQW}F;_~m@sC`Q{L5vq z^z+O^^*xMlByR;$^IzRHL8a`lhrg5Gk)_$MPF`YIt>%5{gU>m09b$7TZJsMkIZS!E z&I!NrBGD1##)_KkzxUs>7ONIb^W0TRCl^31_t3*g_TiGhE{Yby3B)#J4AB)(Qa*Vy z8nfP;IF3GoEBcCQ*CW#v=U2M`xVcE0gzh0gL=0yE9*2O&#v>^B0Jc7)9P&&!jU;U{ z{gP9GhX8Ub&^|CV3~;M%YC4k}WCk4|BRBU~+vTV@d^s6R6c9*j->gY^q_D!Znf+jc zY8Wbmp`Xoh%hlIU!nZ5?RFup7mtW7Nh3SJ(r3z4`(3?hMj=Z##Sc}iS6XN_g`Q#%< zhTrr}OfY2)K7`RX6+Jz6Jl`#w<4$4%y7#|6(EDJ=`x2}OZm*QZKrBKFiA|b54gB#< z&NPH08@$@H>77b;r4Nrl)2mjQuAlokrT#2{G|`)y8iL}|U-S9>GGURnK#pc&$Gg?~ z|BLzl$B*L(gClT$h%~{2fLGGuEJj71l%(&DbPNX6e2#)6CBWtXPED?n0odaN=Kt5CHzH}$?9A(ug$I10NnENpc7H{6^(}Z6 zSCKTwup$i&%|%#lZce{GKJz)7d_1|y*vd*Tq-P_(F=q39+z$Vf7@)=zgWlHF-NW78 z0JMP^26;Lftb)7j;#-0gPj?t?An(8T&->pY;Q$yYgeb`Q^XCUh;Ev_6DGjC+z|r!V zc5NkhT7M@lE8PR4e13r?>toV?Ny6p#>-wie8qjnBO&!{ut&@_Qu8l&55CsK#+Ei^t zdAVahJh2!|Oyjv_#zscR;EZ2@S&uY>ixke`h?DjF!oqMZ*xaiFE_g)VtN}7rr1^gR z%803MFu?V+wEf7(xlIA00FU9>denFvAoNla{Di|46B`?=HPa7?jIxdYAbK1F0db)g zDrDhJMS+W)!f>cO>I!nSK)HmIAud54bcRcfGwR3l4=!RrMfQT0wCngqNN?}qJSnNF znxbDqySxD*?pNwBd=$mW_|m1gbDj7md=~QSq1ieRJ2IY)hjN%flQ%Up%Y?H8n}-lc zlwAMR->^l=Wi!g{SZlUl_*>4%O839S>~5WGoU&5CS%`3SE)KBLMhxvJxMk$6F{H z{9<(4$p){iP=Dy5nXrzEssZuHlI3|z%$+#)BV&P-_)G2mPL~kzMD(yDItU9`OIy1V zPhd!vDj@?nrltx=j~e6yOW~*3#jpAzdsCB(a)X@ zu}Ty;5$7!RSc7ii!GZ9ZGmsTLZ(@gMI8yL{915A!LuzM-v~(nF6(5dAuF(ti;;4F(HUVMqJfFeSK5#D86pO~q3CtNClltEZ&ZPUUTWIWv$CspxGDLswBwn zCs&3?r0(CQiwA#U{^b((2Q-qQTExQeh@m|wypu5g8d~-fm2Y0{p~r*~-V{0|5VE>A%A+N0v)4QR4cv z6`s>~4XK$qaM<^?wL6a-34!(xOk_r$|5og@4!+7!h$ylz>+gq!)nfD4TV%r$d83tq z2g8x)^k4_FNV|&x*{6DAK?B?F##@{lhUKMn_o1Cg6e-pEV=pY7be+gH$}nmoq|>kjqM1m%uAE~w=< z3II&&b$=6fxa`7(?v9`Fz-a2zbh?!n+<%%u;H5$z(UgvbO!DKo|2 z47e>6eun-(t(^r_RcrU|;h-2OsEDYPiqeR5iz0$_2`DKc-LYv<5K$=s=>`c&=>}wIUYs!+N{0SyXKtF?|ELSF>5CuOK(f4Ldxc>;kpS) zt#Br}z4#tPD5$|lhACl{EnA^GfpfL(=KtYCi5Wk9Xf6;zfIs6+M3w!$3d(Y51r5MK zS8PVyf;yePsQ7-^XuL`gf-xiu3+{!h|({YBC7qF@t(Bj3~{^hoj)Oq@kg4!kR)b3?yYGD?n&N=3!4? z-!;JP0S$s%CG9ywCGAAwLGvgHJYJ~DZ}P9nABgR0+!t|$)j$?5B>+)q1qEY(6afHN zTvnErn%Wa=Ht`+G!67kEQOaC{yo1wvbDx7lZ2F6sl|q5U)K18Ir2X~R@wr#VuRjHT zoIFLOOROu!Bz>Vp@{=T;1atbDfY+j$$Ia^&x`|lPR;U5TkTRct+dBpo6l+ z3Szl%#b-wH^rr_;U)3Xl-=rNHT64f~N#V27BNvnaH?f6+W%`?nErGnAJo=x#)hZIi z_XuSOd3gD{psl%C$_Z8`PrwnTFc?kczO>61Z*n+%)#9OtO2DoS6#pB1wuXfToPf>& z<*UeyJhw`oW+@X>QFQ`K^D!WR7M9u)AFI3fXajBUiL-pnMyDSL0%Q?Gz1 z7tLq)3Dje-<{EWG3)VIF(3V;+oQKi+YwEhOiSTFfC?ynzNxD@=hUv<6x!R`MRBu3y*U-X2@!z}Jk89N3prkjVms|MJUV{!r8 z&&2L=6DhMav!Syk#a5~E<)Q*YTaE%ljuTTW9)Sv)kMFkhU30Koug&lY(=0tmkDMlM zR_Ar|dWSxDbh5Cjr_l+iW4{VsD)QMcwh^Dm{&6C>x-Qk%zJi|OEjX@__`J2Xv=uoK z!H~E(t>N3y348-}0A{F7P#h1GyR(q$u)NCTNU@4)bgVS+#RX!YO|6UgQ}K7aFXoo}toFt;7*BRj=<%ZfoLGmcb3l-K z=UTPP?kgBFIwg5Hd@VE^4uz2b-yx5Lz!CH^&24R&YhL#Q*t~?_T6|{d80$c~oX60t z^5n1h9z7gMf(sZ-4|Tv%v~^Y=N$CjRq0y&tcsTpffLper+^J`)TADRKJFa}>Ls@D; z$(QNiX7wZVo%J(QrM7EVcq4cR28Mp|r*YS~dOFTV7+9jRBe4CFc`O(HQ~7XuR%@{q zttsz3y_>L+ty$RumBB#8YjzOv<8M{t1oTnvo!bA!7Jtu=pQEu}YIeY<>~ThKkoA24 z)vTgNtwIT^wc@{~y(7e~Ji<<#K;aOES{V}jzS)YjcmI_PnYIa!6J&Il(U+;Sl%pme z$zU{8N?$p?B0cP8mTRJ?GJN~I{B1Zz(6*1LmUg+?+J8FlQ?fO8PsT#9-6NR*e0B9n zAJ4E{aC-~YR5z&c#)8$bC?j2DFkIpL5c7(pI1YTb!b=`fIC45%U;LzEw67;Qcw0>1 zi&UGmVOP>=W~YK%ddg>Ec*>i%b2iRxn;wAUng&d zZV&L6eakWL`lg1S5)Iwh*c}dvGq&|DUFpy_Ck{b509TI`ddKgaqtLsOW2>XW4VGHk z_w43JJop#MMF5_MjlbZ>EH8I9^VXdJRHvsQRgMlQ;_UBZ01pbo zd`^zRV%#7Oa)&h4K<0Fm(}Lf%#tCNoE32oL=2|Cid-4KmEm+9dT;(tgvJOaH zM$m(rd0UGC9|J9DP*hZgirf0sQv{hnnh)R_RtFlv5f4R`Vd0SKAVyMPM8=#~yyQs< zM)51X)5M%W872+ma@qW{(g7y>X}RtAvAY{~756C}HBPN9V1rO|LV zYrcg1CstcuVkYa8nnoIlPK0p^>UZICm6nR^EkB&?}L;8 z9TYkQZdu3S+mlQlG-p^>R*gHwL?dQ zIS~BwLqfj}qQ3B1D=k>U=6`*|&^@NZkM06@|{79U`T;^sN30{rFIx& zk-xU!MU?3(?@r(TWh?TRkNt_mnXDu*AeWD9S3X1f3-K)FC1xhbt3ya>y5UbNaWgO& zC_|u1BnuFxmAT))$AcyUUXahA5SPcoj~r3P1sS1sGk{U{hCl-T4Hp(>W%;QG>K?<20{=DaHJ5F6adV-yn=$YP%-rE#!_HwK(f2wpWUrv zs6;nK0%+;vLmp^%o>Ijx$ecY(#l&vHoRgQS{8~*|#vj9-Fk{gnbjyK1Ta;J#0V@gG} zQMY9@Wst!jCaEmxfx`3BcK4Rj*t*WuD27ip@RRE?F`?scr`dA*rC`>TVI{lq^wppR>Kgwsw{CF@I&= zjXC{zYxz;mK99`5QaEA~cSj9U9wMq8yeTQmhKih-DdGRMetcvZkWX^$ z+6GvDHj#Gh>eYHEBEcUbq^xW|PCo}ZHd`zGq^+W%z9&W;vs|;BmvS#{2M2b4_{EzL z{v!N=&h%Lz0T;$9%OAt#|1<_dTM#n{GUfnjqA#SWz8)Nd=yZP7KIU+)xWP?ezR|?(zE&4bMfZ~-zKKkbr+kxX z+;{K&gi%e!MlYkGWC8j9!r7(NY}zH`Yn>eqMJ;A0JupN+M^5$g^V26KE_-aP@0oQ0 zcaa)!WSnfT`k%yJ;x}7+wuzpJAnCu}ZjWes++&WnAtaAc(y#P8(E$ zn!U-w;(kMtzm=ikX3UNDRP8ux80#Yfp~hU}Gb0gOeU7fmB*&>j+WAX;xNizAa%T1# z(HF2zJenF6bZzu{yR_x|8x1Ts@{KOEA7$*Wgcrgj;T^yjxuER>?*q)kWCN+zR|qx+ zNJM!rrg+>js;xqb#xzzg^~!Mzici*@w>eL{)mBJM!>6`>SvhsjoMQf13uDl;m4}_v zI@r7d8XeB{8)Uh+%pUg+$tvp_8+|emlXdc665phaRJo4cLiel=iRUaos;M51buwhA zc3C6HJ=B3&xYlH4rP^Ex!&kIs?!-dPSB+=71FdyD?mGgrvj^2`cGMEL&B3YsPceK+ zkj?*(6s~R&Xbip`5W2Vwli-H|z7p&?3&sH!0Ixu&c$qvC8|LvpPOd6^7k&7L1a#SN zgt0rAci8N0@6@#oT_7_c^SI+-NHQRiW6qA%7ufDod@=M=XU6d9uKei2rM|b7aVyCW zQrBXKnEGG8AQj+u)G;=a*u=E2FRe6%=&Jl|{n=x}&CVM9nMinTWyU8W?#+#z&t+Drg#7SIw+o5(c?HXKmuGv^ser> z!H)`DQ=cLIUkepIgi3?o2AR&P|LBVIzrx4=;54LApF`B|ItXVx38)@uH=Tk${l%5Mcb?;vncw}k75jV8ew1xI z?4By2)xslHJj%Ym<2K|U%Kb`oCP+Er_1cxG6zdNw&%14nCse(=X_n)@t(LJ)HQDK$ zI)=_#vC!5ttv_Q$HPO>^#w*a{Cv_k^;Bkxjnqp%FZtU!TXP`Zh>$4)-)4~CSfsN7a zT319@n)-7+EtYcbramfaQ_bu7QwFR%bGg&wKY=Jg^e&KIZ}G90N#tg^Ix{;94YzLE z#2F|bF(GdUznyGt#z*T4R?H;??JfuOT&I|zEI2BlK49m$-&~cp30WV{+}s{Q769%o zF0N2;U$9zFKdW>CFA@;7F|g2q+dL=AKnNlHH#Qk5?MhVsk5Q-G$%ao!2@ejC`)!~| zg6oPFZ2BP*PvM~j&Jfqa#(Rr##8Is%wSU0{cGPQbmB-ifjeH-+yvG_n6PG_nKc|pB zGAqRCmYF-&IYYpZuaIY6SUz(K;0ySxTZ`%H&r1$UfMU74t;8u8`U+=nYpctmzqd&B z3i>y}Hzo&TT==VG50xF-{OjlsE~yS>;eaJZZgrY6L2>IPe%AnxMK zl`JEy`r#0vlB}H_-)b_RMlw(9aGXY8RjD5@bW9an`;uul)Arzo`6=GYx;G}ozW}BD z5AL7+@xg>4RCIf(Ki1!JY+Ahk=0z{f$D#dr=&wxjtn}Z!*-TL0MPy|#o5h;drE_Ln z@CTw!eh$VTA7HBp1eQQ7@Uxw9o#0#4|K8++;BR1ZMkpo-IR^#-r7}_M<2z^Ub(6rGtR?tyC*v zt2Y(K?*Evk3APxP8EW9jq{!-miIUEDHs6}YK}3fQ@n0D~|NAtPv(*ih-!e}*n#o&* zDh2(+zYMMPu@@m8aV<`*p$Dk;T?bdUVt+@$_tay2UjIzI?KFPQ=X@QM7l_7o@3*e;ouw6nUkhG&Z$0WVh=Q%}Kjl8g$e4v48qDo2(97!1$f06C@im16I^7Oo zdGGR8QIR$_YCF1Sy#+P0a&t(Do-6JVl}>2WcewCkyiqAuhmG(6T8B$dnf@fQdoQaY(~B8W%% zBbCr9lied;jVom1ikUbRwS3GXx zJ;S7M+i8@0UepW=Gp(NfSzIhZNsftgT5s;?T_4Vd-tL8penMNNsS3rznV!Tv#fxzw zN9DohrmCTBCpIRXEreH(qP{6Ln6Srh#GuO^uk&LHXHfh^5_v3?=MlsV_;Ok{wjhAv zyV3$r{3*eG0FeOSA*vH%aIV8NABL#paC!YpK+u6*cn|?WMbps>I4gFr$^bz@$MoC% z3onQWK&Q3BzBVn1Pkj)-3k=a<2Jjw4gtcB3tp&fm>XU;diqF4uw+^9z9F?f#y%p}OuL zuSC-)RE-VZ6G9v9l$?ACi6OAh!?qqv32{+SCtt3T-M2QaV+CqUY}jCUadGDLG?2`y zGd9@Zh{6c-$SxD1ipHZK`^X(hY%eR)`$%c#MM9?Mgivjq*4$AGrlFj-V)lW>s`K!~ zNU$L7X5UV+;jaqZMw&(hd{ZH188hGFN-d75GRp%GJt?5Du3G&X#4sMWLRD>Y;Defq zY%l_L7Q}CYhkpUtJ!%@OkEQaLCO4t$E?11C&q(fh*-Ka;oKS*ZPcPDX=qfg2Z4o8+ z92FnWz%a~PMYd3)qB)f-;^KNSj;T2sL?6fi(FU2+uuv5qU@x>|pig84vII#wfvM{= zsJRg!4mlzYu*ZoYKLQ;@1nLsFPNYHa1s&z*&u1F^F!I#V%7M(gw`8{b=JYkB+ zdD-NrOf$`dAC9lDpW*Dd@A7X`lxv<(g|7finLZ3tiM6S)!be{@&s|9+BOSA6wH$!V zx@wn&H!P-YCYJDyxOTY@s>iiq=QTyqPG^+vzgMUUH1O2@wcvPtj4f1oCnWg(>V zLW$6i;`+(9r8T?t%ZH}C4Utm}O{yL4?}TV(X^u8(Sdh;*IBc?<8>dg|MO|-?3Vq+z z;Hc2n4ljOCy$T#^*ZeZKy}$|%1Oh~&LG0I0P_0T}ii+$$G(H{&8{KX6?vrcS7Zg3| zReZQIJJMm}G^4a)8yc_`JLuLp@#1*^UFFxo`tEl@QN?398;?_oc-GqAN));~MV;sq zH$>{_>^AD?n0v^^%RberOzI9&sC@gE39X5S!PmsM9xV*3o#-~`LUGFc z4*_R`9R@C(2>vdD5r6d2{#{S}3v4_z)S%R{0z(GzT+YL2h!HBLXowSZE*k3ujl9^b zsjn}TrE0G>zQTo|-QC905($8W+%{~HnebrtBPG`sxxk5u35gt`OWU3;PaEFF5D##< zlL=LB5(C4ONuFuAtxk z^Ij)&O;XGmVteV#=TT6$d?5VP*TC9rcBPVLCvmmh)lLmx_~jim*>r1KNYZyyk$$az z9;&~p@|s=VO#(H75w&vm4e!8$c`L?2JLAQAwc#SB|f zKa7$|mOSCnc#qOvYw}!miu2FrPcLqL27lJ+x<)R|$Eht}{S@aexy-FvW)DI=GHR>Oh#QExBM4SbU~9R9Dt(nZ9YmGSYrV(P&Qs_ z2f-5AaZ4FHZmjxna3fYMAwvx=T;A$kfu)qpI`!B72DGx!I=reYCwoxH)hm6uGzz*d zb>C8i2xHA>t1FM<2DQml>-%Ut-bz=ar=+`va7?DCtCDo%%mo-rt7IS3zh2>nrWx!S z`;LvB(6Q)Gh~cdog(Ypg^?+_<`9O~}Mr+>pTL}F*_j}KI$mDRF3j|M;RDSc!a~(pc`4`Nud~cJ_m8h^ zw@%d{^I@d#fl22KXwf-Es1S*JfcQLFF`wS?NRh%DiS$B52j>Vh3nU|`|vj!}Ez3G$C^>Rn1u~E!O0{!l)KY#pE z3HI=4W|tN#`fzyG0zJ(_yUm9IS3%}?v}VM>zxlOAyOHt>OTu#rF&P3BOON;WE8_-t z7apQkM!c%4t81)af{81>W1`}ir_}*P59}T4Y-|FvQeAsM{@z`PTuo(okx1e9jowk- zY#BS=q87NTw2yypL%s*jxXbZHh~vrn9t*CdM1K%4=~T0ncT4+4s9r(Nr;}N)mk1D4 z?KKued8PwwfIsR~eM-2MF%gLQ{8CMg9me3!5?i>c1%@>zm4~d%4`z~Rg}TjL=@(PO zb95F*g!`=Fh~>>C(KSIQl011EXHArIO3f09*bIO!tA8>v-b2LD4Y=EuV`?WJFSePP zHlo}1d^WR&<9=%xVOiC!hjf#!F5tElI~ zu`L}G>)DDi=rVG(*CZyN2KU!xP@=2#!>v(I&JZDPygC1IVDkXnim^}Stsznk)7F@0 z%aqKDsjkY)&7JgKb)N_c)0%A0wP2g8wwd2Xv6KZY`hP4H2j3Rvi3Wzee`WwP+g8MG z>S9mL+iw%T@6LV>0vhV4G1ZL!(CPXh|HEpro*rtma8BoI&ZH|d>wy7ip<1oo_bTga zX|2fn8(#TxHN?`T&T5Bg=*d*GNGy?a?0i6?mA!qAEg6xlz33lGD+mY~V6yJ2OT0QF zL-X!6Jk~FWsgbq7ia5DA!CMmXV{E&x*V!%Dw)nt@YU~IX+jl}P==7z~q)!<#aR>dx zp2x!+r@oC=MbKy!CejBkz>23q(>DjXD^6h$d-+!LV)b6+WphOZQg| zVoI{T>rnZ=TG7)Oo;&aaLa!to@VxhPnKYxA*%<dfv5;G2nWM%z82 z>R0^GTNbIj#mD3kD(psMdAx`M21WJcibiQzfKn*aFEFn02?q(@Ht(#h^}xl$MuL*D!3 zC68dGjp4Juqa$?+h8D>%%O_es`uh@@N|;()Yyp$?byl5IFsy)_6NuZs1}W|J{sE|X z4yCEVG2%}1SO@#<#ta`)@U|6F6gh}|xfT{=d*>s;nfsOtKkiKtH3p}4)b)*pjyz=0lQm77dh?@L%KBM6Ha{@|L-^0x^>~nf^6XbN5!rU^*!iO%sSg;S{M5J`lY-As znhCBM&OP?Lc9Ak&uE-CfQ^wPc{z$nsKjsN`Q>b0Z ze0_~is(HbShCq;&K&#jM4L;D}0DxiunGX<*6zAzpxY1KAltBOcmA=vYL-!G*-`STj zW~1wdOK%a4SlZWQ|GbDsXhgX9#z(y?Sed)~4s5Vu2Zn@tftNzFy>fmemmNLpmjpQ^ z1rQ`Fli3i2+`YgrDuaL1*5(-ZKML~O<5{VozUn5k`eXb{3TzYJ%HpZ3l}#-w?gYlT zQ!>M(UtP#O#>61U>iEb`p))neg(1RY&7*-{ihk%C z3H7bz-Y8>D zydV@2QfeS`EgQx1^(Hjue=rhUXsN)qrZ`#j0nFdf)Bgi~X+5d14U}xU9~k{b${dBs zLbXZf+UJYSqzlhv*ESB~laZ3aKnA>fjV)d08P719s+&G2X3c4qbV<7kFiBXTn7Pw^ z-UZ(2*V7@#(Gij;2`*lGUOf$F$;HjQ4cl>cPWFbZKr%`UrLFPMxg3lp?8%Itnehq> zBZVPVT0@4@+TTRUyR=#-l#5*Zy8vr>`*(XheV*m7<&i{Fg3V z?72pxFAXz$jluAR!;D}?stOXIP5ohxNaeycUhFiNZo?NJ1t%pu$oL%nn;e=PJ4KV0 zi15gXiRm}NNyGq)))(ipPzE?7Zi<_n#_znv28Hyrhor6HLvpL|-c>};U1DP4vT^tAG*cW*U&I3$~cXoE5x*Y8UG!6mTAk7|fHvyPaQ&StYTA%pG1-X6+ zp6xZji;Mc*>_yQf5(cX@Auur@bh3Y=Pa&^PEO2nn+KP)jhCLWW1n>R*u3??|0{p^r zu;~5;l_?ww!jhaiXAq*{5~nyU3+Z4`fgzRk_&@&aF$ydKS3og?PP%jz;S7a8MBW7} zH1ZwbAOFY4_OAo~$AkTMfAMgc01r+BJkDXol^;&{;KTrl>qSqHGfa7o?eE(UdHd9d zwU2;)FziY^4G!IJ2*Fbb8-Rno<@vDopY=1=_YYL`?^Ey}-brFNY?uZf9j7?J`g>6? zrXFU(2KBqE-6JCvMRk{v50w*O?B}YWOM{w-7w?WefpG`;*^PPA7ek_=Bq0*v93f%$ zikvaB#Rs(HVEzM|A2}BeIEZ0@9DD%~c*ZD@0(S~Cb298e^FHYc^7S;gyzP`rScE|Z zX3m8F!k;r~eEWYtlRk$<;LoglS`Ly6vOgzW5|G{lN94NB+;7N%f>DRVU)=Z8*0PL*u4fGQ8YpBb6A)d z*c7tWzdctT5Y^UB&?E-|9!c(if)u_Tv35Xk2;%bCwPV3dYj(WsDg8iQ>(L#)5cHNf~ zF0>k-_cc_xUU>^lG50fXpL0q`OAEaMXxcNU+lG^gBd0G{zVR6Hp~+U@e){ZP(gckn zW3XNdxn#u&ffeQZ;RSlK7_5!ZI8mEU2k)+Hw#{wyO#8ZTW?o)*7v#C!SXr8%|8B`+ z3Wm+l*jRILrnBmHM)T@EjO&CwZQ^;Twfws}YC2ZL;zMm1+Gt+>|)2E!1?o0**4 zAe`{o&fZ=PqF{h=?+~DrZ9AT&-esp0aCqf%goE<{yJtq=UCq1$uSc;Pd{uD`4l!J6 zlZvJU$eOj`1Sk=yAjB}HKLW>h6rdc2bb?G)0Z2H<9Ly>#11r>hcMyU@bQ?wQT!zO( zJ}jtYl$59uw7vD^PW3t8;vEQ%3)Tyu67n~~u1$KvYl!6vQI*C$&LzV6QTvLF=t?#W zmS$bu+^X&xft)&|3=w?({)91L@`E6*t}OnVS8|}S`uoF<6~ABxOD?7jfU5#5fqmeX z;CS$0;)6DTL?2cC?|!SQ(hPbI4$>aXy2En?g~auBD?#^@5v$SF={sF;dJe$@b5|T< zoLE=DCBQ&OXX%SAo~FdXX`^U(c%@@*v!Oz(%=X*e266-F)H2|(>DAR*F!6<490mA} zf*&C%yW0VNen!T|67Zp~dual}A9@$UT6lHVoGQ<$bxM)L4b#%rX1fI)rbN(tZ*L0M z>R=y{PTt`mS{fSQykSt10^~LCflB5J)}^E74nZ88J^rhQ&yXBP3L!Uj7R`!ZP#E_9 z`gMz4@z$kZP^{mR7ODaMr4JkoTbpyTgIh)HO(u|?<}R=sx8@+QvG4Lp?Ln{8!FCLR&qO9I2fBp#_!h6jnLmSrL*S!BB_0?PlLav~;tIyb=}_#F z3ic*L{1M;%OJ@zOtrb|bs%%HlK-R0~>-*8r(qauUkWP2Jw@!WF2Bdiz&}%F?ybf#e zcN^+yq=j%TOiUD=oXT{a+r}lfqTxme+lpHeUIHlX^c?jr4$doW`DJP&2#(Le zV9ceZq`=oK3zTMLn-r3q=MVPXE&zog8kSzr_24<|%ypB3q8I!DcCcT_C~QvTWIMY$ zl=s@l$=P}PIm2xMZ}9CsbhXF&>a|r4%A9_aARSHk z7>XeySL_fRS8Z+NDB zooW~Z%bP^-5mPs`nO2>A{(((_8&JF(Ja|Fm=SCJGWQ7m%`u6s4abCIQ`oYD3V@dzX z6@K}VOt*szzlJ3aAb+a + object + weakrefs + dict pointer + GC info 0 + GC info 1 + refcount + __class__ + opaque (extension) data + ... + __slot__ 0 + ... + > + ] + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_full_313.png b/Objects/object_layout_full_313.png new file mode 100644 index 0000000000000000000000000000000000000000..7352ec69e5d8db2c59135cf39aa0e6f7dd323cc9 GIT binary patch literal 16983 zcmbWf2|SkT+CF@19*u>}Nu*E;nWaP&B9tK&l4Q<2Rw^V(2q8sfP6%a6nWrRE=1i4& z$dvIr?ltVa_TFp1@B97NZvWQ$)zkAl_jR4uc^=cn=cJ0l<_%062!hzGcuY>6ASitB ze}#2x@H-kZubl9&H3r8O9u@+-z+PV+k3ex>&0>1dY*{wj~_t2iE6*Jtv;{l?+k9a@~W zCr{OF$$hwM*Le=kCuM^(UNY0x5zTrLwKT5MjRySR@;XDB*rvY;yS$qjN~WQqprHv^ zO*u_@&D(qA+zDDaxo3A2Z+Lra2r5$31nlNs&&jD?%q1%)mk`S3?R{)(;3^uLc!LKV zoSd5f^~Lpvib_fj$JB-{M2KB2>A0Y_qm)aw{*dczmzSvXq)A?D@WY3kyLWG}u&{6( zdHemOrluW3bW908H!fhl%ZvS$d8>$Cp7YjoZc|%T9!ol@mQAUPQY_hVl)093;)Cw+Z56L-%RxO%u6}gu#*G!}f_wFIy_?f7 z#T@aXw7PoL6Q9rKHmP=(Rf4|LZHD#b*__+j>BjFnJ3F79^`c~bQfuom!g)|!w7LZr4=hz zuDROV1o{pT{$H!7v zXa`f?Ejd$lsHF*`2qXsDbFwB>CvP$6pRBNKd$XO5jjbZ>V&y|0Z>I;dOY@__B3Bxd z)GAn6SbD#t8dmJj;G&_Ye-ZR<#Zjh%@J38;!prcxP)5tjW2 z4g@#x>%2bcKJ)YEDR1F{W9sYc zz}UKB&9Q+o_+`(vWysv(?4PsX^5J=eKHQ{k=M` z*V+8s^18Vu#$S2^>ML+==gytN(vzuW%t}bujgQiC zlCmE<+Gcln1ml zG!CUS8FqcRMO66dIZeksZ?!)Z3juwL9CLs(YclQ&?E@j<0WD zq(eu1Q-kum<)s|S@gh2)({e!W`B*|#>& zjV@mFNlZ*kI-4^s@GjkG4;qC%zA%?%-qb_#lTqcxwY6%wxw%~X_p3L)JiA9zQUz`ZzB%fQcJ<4NA z+j(Y$A6wh~^(#kye*T2a@=uux@!Cbz9h^3n{%5^^=wG$AmOXlOHNmlcJ4NB`&FNR= z>Q7p!-1tFZ)l;&$&*9RgOA_*jH*XGHrMBZWzU<9gx1K(GwkpSd`0&MxJBfnA!j)(t z8CPxV?~l3A%K4o_R4`#&S=LD`MU*kQ-h;%{DG^q>+@V3a;ICaOjCM#C%ran-k;cV zK=J-je%;KXIJKA|)-^QR{VM((3DzqA+0ZAioyoT46=K?qUpkp)m$O8>dwOc)Px1V) z72nBroJul!9tB&w(x5R(o0{h4?b|K&A{$yjKtNedO{otZ&(Nb+arU{{KHei? zQ-e)MqqNbTU1>9G$A3O(Q$ck)Fb>Z&fY8RP+*g`wl<@% z>6W|lG3r_;PoDfPil+Xw)?}>Rp_Ag;{>$xHO5zaqyu-0)^K0`~`j; zY;mUZKzJI} z&55~%Uq2_t3aNavva+~%c#bP7`mjj5p7~H%SX^9u6zwLaURgzXKLIJYYW zXUEA`k^A7m)30@sZAaR)d5(_A!+jJu>R;c9uQkyFnAr$44Y4aG4%@AH;j=en7y}2!jf|I7-^h)Lhkp3J2osMBCMx zt`a}yGdcGB#Mi`>{0~=aq`XoMOQRJ7xzD50ijz+r%ugi9E>| z)#IZc%Q+=>)>p3{*pp}(A*`M6Dwe-6aU8RK0}Tz0Ny}?q?4I|{n>Te@Keq#|*@kRl zlVp7S`0*?b~2yzkyE!lVoc5BJ@hKf9tvY9`{dN@4@ba^<~y_ujZK ziuZoId?7t5-)(*caeLFY8%KTVFHH27mwo&wuc^6JPVVi|*Jm8>=EcP90KmFFnm>;R zur25<4-5zk(@vQpzhAA15TlH{B>Csj(VSe|+=u0;>F7MEq-US#TL+i}S+?)|`0*p7 zl(P^ei|DmqKffFP`1JHszb$cXyr;AWi|hEa`XKog-52M=Ui^9t|z6ZoKZ;@#CKU{#3?Qp%i=f?llo&A`W3fo0>W2vAMguTRAwC&UAWEIXF0w zPyhDq+bNyW>=JEeDk>@#3Hwb1aNtdfm1}mLmfOk0)3k>>jh}mcbdJfw>^N}^(>ShK+dwbdcKx2yFN<0&-}#|1Kh)OJ zpr&L|HTOe8yzmX!SFc{ZYtNn|rlz~KlFq2bGOk>);?;%1HND@z??5-(x^=6a{yTgz z22AV1r$@X_s;OP~W!P&_{$M9HP5LUOnb56)-|CNz4{*0`prm}Lm8_$vsTr*3G}ZSC zXfE~3%U7>X0-N$;64LYV@QhDPJSZzW)|~H_qnNj_;2I`wmoPg&?~C*4=;Bg&b3L8W z#|W_%hlpdqntiBNE&bEWe7cz@KG)S%S9spIQB_gDbhduyM*vFS#pkiHzTRcmSFSZ0 zsE_~r@~nilo!wSdRn`8Eyd1#AmLa#8xm0_PM(;DC7Zj;p2kcN-@oLFOOU}sS_aps( zq7KqMkE)D}j5v6B6yD~oq!Bhl>F4%jo}e}278PX_kEHZmOO@%cOU~$dZ?6&e`ZWHS z+I2CtacxI~yqR8ROV-42M+E*SCuzTKH`vJQ-*0Peo$4Se=YWY_%gT7bc#|Ah42^Lm zhVl*UshomB>-Myq7{Qlx4R}d_pU*ms5R#UZy1mKCi8-C3FLJj+Jo8Mf=}c?3TD9YP zz5u`eDdz}Dr_UW1zGxWg@gy*_{lS-_#YOFh7;kT8UgPD)(Vn}Fc z+zFE_LQFv+A>n*``S^;^)M>VCS(U8i`;cEZ+GBZX7JNfZOY1g@2=wX6>(^T@U%sp) zbMKxo`{e1YIujw2=CqwazxeVLdHN)Log8juz-x?$E&NKGo3~Xz5-xlq>v{jtBb`C< zc76A6Rom{eNihMe?_=`wzIRVVY%XpvyYKsV!;-t()~#E2Ej|4JU>JEiSFWXuD+vk? zzNHo;|K3Ds_eIpKaQ-C?g-oY<5n=8|Y03x?=?{&UhM8?a`WWg^71-vYMK!A@S&2t{B5DjpXDZg{v2Ff5|&5dqVjGleyGR1Sbfdx z%d7X#7TVrk#Ul7ur%#_6kGn2T zHF4WjR#ra814q~OgMFlw;HygV@|HhyE%K&WQ7{x`r#v`~4c|zh{rf38Sy;R@$K)NXH=@ngzm3AtBnq(ARS(TOa<`?X zrfSLkI)_#KQ6C?us11tw2`^=lbfiYt;~sBw-=Tmq^Y&hc9lik0YsYqwkFTmXK^o95Kf-S8wjdIczrm9_Qt zoSdBI3CyOZw^8VTZ{EBC9cdLmL5EVKuS_(|a~wBF9!IxXNlZ*k3`o9we}13S@X*jE z!Y-v9Y>3`J=V^Z=ZE2vEgt^Q%r6y%juVTg1YYgpP!4* z817>{sjq(zrJysY`;*Tfyw%%V`&Y3q14Y{j`i$bMT_08x2W8&3K5EInoS}nSEkH8{ zfA(sVWVM*C%d$KdUMe>LDZ^CF%{gxF^{EW7ckq>lw^3b%UMmR^EPUqBcqx17m6W^} zx%PH;#Ti$cf||@*UcXP!OdJqWadHwvbt+f5FHW6~knv!{eveH}3H$Jfi8Vj70D@Fk zR|giYQb}z-tUO4MlT}fnGb=q#^7|}Dj~5$K;R?aW8iADwl_4` zMvU~512vMKcsHPC-q=X9_!oA%xsHN2Zrs>s-7RN^!U9%(W7|)w^-|9#tJ9tNc!)sq zl+F1xLEWmyr+PjlhF4v@c(EL#?AhhI7ZuMVnGae7p~xsWh+ZJZAQWkz-Bb-GXOR1b z4I8*bM6{qq6qlDPy1KbttO^x;5E*H{zXRaunZsyD_qT7{OG`^>uD>S4i?^XEV1!R# zJrp%Gf__XngA0 z*O{nZ`uWR(gK=2Ke7bzmhw|gjPy9f!h*$25&JqfTry4+lwrtsQW>ILrsU%>s6Zp+P z<*;b^Cv*A7)>5$*y}kM!GR~{>?^n2Z9rYvj(s7usBIO{Q=4OdarZ*k`T{Y{DSQS=Y zR(425<(}pY6=(FkLghpL(KldnVe2U=Z$q2Nxg|s+EQ!5F_tBM)p}{Y_B%PdlJFmQR zs}cK8HLmuo&DR2o6Jh&TqeO8kC4f3I176cuQtEjYehl~t6P&WyJ7Zw)<9Y*v>wompE?N$=q z0`AK}DM@wuX}o3Y)~+A5F=sN(wh5fezuR9I+Y_L`7C*J#9H(8@%4%P9OiZ$V?0+!M zq-=w4_A<*02b`ye*5~Q_Llouy zCQLtywp_`UM)!qr1r{-zRbczqRANt6gbQ81fxq01)&BCvI;OhxdG>_|9BEIXynyT~ zIWdJY0n(ziKg9u^vS`I5^)8cgYy9cQd~igkQ^H|n6%jZStz(BdSaFs)%Cv+_*64`L z((Dd`Lr7@yUMrw_?SLpP9bItwMReqe2^-%&D2(s1MO|H86fw2lB)d`^ADAQ6Ok!7H zBPrw!Y@w$&_*NCRZo`Hf01_aMw`D!oY!?>JP)TJ2zP{F{s{k32f=EkGA06`JNCfh4 zK51bgKor!~rFWRuM9EMUh{W2GV2S2Eki-=}e?PyYW@db(mNz@=IMnu*4lqD7`J9&4 z8X8%7`85Q`-n~?q@g)1|?(PPYzXn#i4Q$=p+q(b?qRy41qeqVtB=SnX2pxjBhHCjq z*|9opHCeN?!N+wH?FQ=0djl8HwyFl&Dw+4FJQ7+jrOs;;HN)1N?SBs3Ys+j6_1ut_h$@RU@eQUxbJx?-_M5e5XOu`wN=gz0 z`nGXpLS@3>r#vXhx|L9^k~W8o zPy;(}#<-+J#lHs+cx6;h+u;0AfOqry_3y(i^L_GMX1b%67h;R;A;+9~YqL|{kLkJV zimX_2_P+U5bwgKELxW@I&edp9YHDi2(==>s_Y~OOw%|LReEDI8MZN;Cizh%!l(~U| z>tjH5h-vFE-%Ekc0wN>*&`4sSy?rtf61DDL1)MMjN#!X{n{nsnGq285C>=je9!jH2 zmkhg$e1L!txw}henl<0JwUHqjr8Wy4W9YU03;MnKcd_Rb09Ig;3XYDVHdn8b9j*7< zH-o;4;EKyU!az&p%Rd2B1_ak=GnJb2CVckYWuc=xFR&noR4nJaPwPTfIFc>|UU z(33kp22clNVEl>k{NKzU_kS=rudGYlrd)hdpW@+N@3xXz%9z`_!W1HrbQz?{U%$AweU+io zZsKfCy)=+28W~A!N6o@=7i~1LiI<;$GivRqkr5Xdhjn++`ox!^Z`(11$D7_QZH0fK z$>yRov-ZUmPG;RVR@)!)Y2Q$l{1RD+8=^^dJ`Q%Ip z(UU{Xbe-<={3}+hIH{v!nx0=S?)rl|tm}x^NE>9$@h^t{T_h?7!qgmuUh%%Ei9u#@ zGH_yD9iHSV=t4%OvZ|^=&0{HRe6)+b&bR*9s1jgCUl6vOiCwkma8MM9gYRa8Lqbju z2GBP6Wy~LdvtkBRtQnV(pl*Lc80-N$Nlr{A+VRQA$~U?eZEvhk&!?zR+b(xSpt~&2 z_V^RmK7INmVbPjusF2um?8FHT?hx1}7{mMzqr|SR0VKHX<+b*%{0=m1W$xm{L64bs z2ja;I9bn9}@u$w5@eVt1iF4Pkb+xJB}X!rQWP`)q%FVyZMWG^Al>_Qfu3mzHM3Y%PM}r8yW994spb$&i95tE@b7 z<_sfz4PV%6fRbNfCf-g=+=DhbhE>tY6EGs3RuPTX%TEB?!(1P*Lk8E5ba65)MyqX#b;4Kw+_?wyr-oH-}#s@v?O*1e( zp`2~Yv}iZ=JXWqxzRM5zR4a&~&Uf!R%nJ((DF~3tw96-ip(=ByL`FtZJW&a$hOP7D z%^OB=>Q94H#+7+_d5}DJ9^K49jsoS!lH1TE&tzGg%}Y7-NZf8W3oC0XmXc|3aoExW zLifshtP;k`aawhz8T%MF?b^Fn0Wx{>#6aB%bjBHRp|tJf&^B#)$qJa^xkKUJc}QXj zQ+#LDnSgN*cDmBJ%#0MvEdNsV9Y&qKvguR0%Ot!3Ffc>A&A}APHDMv0{#6SVdrVRB zx=QR3a0#iyJD9s>@PuZy(ML(elRVW!LuMzUq**c8J>_HG!`rwY6m-*HdTt%54(A^` zZT|ko>RtT&#jh{d-N&543phDA2x#_S(k^bK7tr&@lXR!oNT=7egz11oxzBd3g$qu) zAwW=N)z!u1go31rMzA|JGVVHUbn9J4!Nc-*8R1eh<>CaD7y0;EwFdo}1)$rxYu7PA zaWFSue}DhU!6wxXr|+y{b^}4Ct;-AFW%j!-IFhb06hENI&u}n&e0>A&+_|Gus&ecY zHNhfZSD>IjV_o}cl|4TBU^9}a_SQBOCGn8RrI{^l4gl5oN4MLOB()<;gfd!!5f>t z4L3YJX;@j!{kwdaa?zx12kO@YGBEkA!7nc=_DqK}_3N53HI@swz1-xmEady|ZU)fhXX&hGp#er$h+nOXy1D zCeJmS4@0q1=MfZC>(_-sn48;X@C)5+A7T}x*kK~XDK6g9`y)oY zfuCP-jm)yy=&P~~C#G#x8z#-_U`@e|2xOCTE_*EH(pwR{=RsH)FRO(87JSp@j4L7P zz~84yuK#z!UGr5_^pf4gv!tXdgI~b?!Gc+qodV*KDwu3l2DW%-o+#JP1`GeMBHk6} z#1RRQS^6l3q269D0^C#q9>MncnbCt+Dlh4^mJHW;N!t&F!dq3Bl#&Wg8yFsd+A>kG zC#4MSl}|-QW#ZEjs&Yt%02%IT!zpvIx>*)Vkys+9;f$td;E5!DgF-<|L$jR>H1rNM zB+?IpF8V-Mgo||2exxnFUi-C4!2Xit`7hw1!8TB=nL1Ha7whBH4E8Y=)zzI^L&e6& zqY`m);N9}lJbhCv`hVu)5y`JWURRW9i4!6kGy(AY))VVDZ`Q2%O?vEwVlNV}tFJGB zu1*cj)5*z6$;n)i^G|V+TS7DA(SxF&H8Oh%BS^yBeGmbG;-6z?(U*3-z8qM*#6+9H zWj*Mho}NzJS739J>u~+4d~vPugICEd(ynd&USX{>kponwXu}VJg1AEVo_h#0t}kj4 znU50#4O1&4TcjJ`UcD5^W$ogcIr~JRG2PzdrCoOm;xjl4`*{aHJzWXna~--p=fQ&v zm{7-#9V3YJ`9+*26#H+^8Dcl%u+thuxCo9v+0k!uq~njLUIXGXV3gX6VmgM*1{5JbMho+$-$9SeE4u$myf!zA4-O32xceZi++ z0Ws*zM@cvcs6@aU2#99PfVX4EXqKc6q5?-N&=3OH)1F7PT>u&cFyZa zvY0A)2$b}_VAC4U@+&w~MBLAd=?#Jao#1?-RbsHZFqzp2D%mp z3@lxEA=1EJ4^(56X$H9k=T|sBcKZ6uq@+!#;bga4rfPL-rA$Rlr5^0!Zxa7Lj|~!Tw6?b|0R}Yslz`bbWHlaCog$woJVXF_1?+@ELO_gcacv4`Qk3fS3yG(T$)t8(fO1HANm)FyaP`4cKE=H6| z4x)yLMVt472ejaxJz;**6x@v2bNOb@e@Gn}nv3kl& z_8(Y9>yb9Gz-+V(AZZy>zjy)DxhD!U*B=>Y9655NxTZ$ceSY{Ltb5u#{k>#IE-btV zj%Yl!@n7^?)TP|fql!vOcc7+Tbp0Ekai)J1L-uoygC!;fori}9T=#^E+Fqd$lt$KfAoW{j9!Y*J_AmULksT}jQccD3yvMRCOrSoT z^MmoH-R$Zh^C`cy(8(|jR(uC_G>y3LFSB%pG>D3P==Ph|??UJzaeUxKYfm@=Va16% z5jNQcwRvf_=Qn>!R9w8m^4+3V*1-@{yQaO@r+)s->G@dh4z9|ve*JoNOxIBumzio& zRP1)Y_Qgy~`2d4G|2WKE|5w#p|E%;U%{_PsbbOzbloXJo{)WU5pnW9({a{Apn#XQU z(mw+1FbRY?IZYL|p|2j6mA&4eDiVGBxpTKOc$WRWo8CzmhnK2c4kE^xz%<%!JN8vJ zLc)Q>LL2G%-ZZ7+_an7)UtdCwCsg=KW8S@62znks)Z*`aS-<7STQn?x2W?$acHT_u zuUIVBaLF12a6N^oVX;$LZnLV5&Hlhu%@1o$Zq}L!YM(Qt8`S19! zew{|ArE*N|8DT|5TM78<^5_2%DU#S?Y6PnBAFBqCMTg5T!?DnQ5+>`!_5=hN)u;zFqSFD*Yg)eRg`{nV-I3Y=?jaytmha=gr< zRvgG4c|ug-*suc(BEeIi!zSzsPCAqN*Fm&&Z|>h>Dit9k`W}mRL;>Ov!P%7| zdrJY2d``uxZ0F=$0hOj0j}$DY+38$AM#el!U+eI1TC)<|TuoiwM!{{11t)x$UQTRo-0v(Ej z4*lnO+t<;r?>$A4XoV<6&v%9(y#4*j4Cgrdcnq9l42~jleJ?$BT0`tRUqIOQe_l-> zR3`HT6qgJ#0KC>Bl0iDeWJV>|>1R;rfQ~*I-gS`D2 zQ~3RG+A?T(FuZSx%Ia{g>`1gJon;fEyoW4!oM5*Gd#7i~4LBPA8VLWWcs~qjBA~Ec zS(Y5px0C;k(_MgBG);NSJ1Hn4qKZp4^P&*b1#pX;o3v_$!GVE>nl67wA4k$UJ(gKL z76;E+BPihFHF4nJIax|)l}ae zg^ZSm!YBCAkk}-(ZBXSm%1A~(YDzY3*Z{S0MTgtWSy$wdX3W|iwT*hj>i(8rI{uJP zR$rfuj0?JW*>pnv_|3-oZr7;{N0?F2VAaxhkEOMMnKQOSx9_1K1gL3T=lWxRG$u0x z7+)B*T2}yi5fB*Y2?Ffm_4&y@#glpe%~8>eKed=kS~EzBaKfB`XMm5iva>siaB#rG zhsK{R5s|`z38G47XJ-$^)*(kki5`;fXxfcI%qU`p1lmfHUjjOl4lp|E>5|7nmuX2= zp>HbLYeqNIOs#_5`N#J3rJw(WFb+&gvm#W0aPT1$lepqOyZ(1b$8gtZI!-rVq9`z8 zg%SrO9yF8tdw8W{zk))kvwwJKsLAK$eob)S}FSG>M}w?8f|TD&mg;z!4S~jJLD8b z_aJkT$ooJ4pOc!;I(tg}eF5j;b(Lf4Ktz#)zRZUOW&p@RG~YNi=k42an2?vXIzXBP znf$mf{u{ULmywi2jq4dfLCwhb!qtc1H0i8F)n%X$YLIErKPX3VxsEwGy#;|$&R?1v z6i@jcnbZUVz4{s%y!kb6-kEvC*O?^Joui?`C*7#po)O^#JOqnLc zcf-_L<9?%|RZC(}a*v~(%f;3y>gzvNXM<-?^rB%2#(Vskg;CO0SAT$>T548xDauU? zQWF{C;bj)Hc}Cpd%q$XGkElpA5S&gV*JI>M{|MC9R9A0J*4h&(;UF^j-5!UHhCnXB zCjH{n@G_hv8iL$Fu%QVAXe!b5`0nY}$wbZF(_Qoz{gVc-I0V@vI1NTfe{*_lbLjT% z+wGKSr`j_I=vtGJACkmX1NxA5C39woxDmh^zKYQ=TT_k=D@IGZHI%e)y{}W&xCBT? zoVV|h`A2*vjI{#0$?Ns2fkYkj{5oIJ#3AslCNSK?Bede^|Cu#aYQ}jb2skj@c6qtE zpHLkVdO6EV7)DS(n=d}X*dk{;oT1Z$i&hD$&oQpF+n3R{1>qyP&JSmyF)<*bn`%-U z?UJbv>ytZWadDA?K>eJqAY-3!)z4Q1@s^d9om3G-9D~F#871~Sty*UdbArKWAn};` zDVQ@~sf}KZhE_GubZI`>kAFUGb;no#foC{=NO_Q%9{rSqx+BBGn@I)x0PoD$*sCw+ zz&)&?g6riv(tyL0XkdA1beR+GC8UmByLa!8_dsNICn7nt2muwex3?#CA_dW2JExhb zuS!7GO_)O7Rlg@vIPsmC9dJna@`RT@a7Ad1-LWui& zDbPEd6z8wVsGiJM_yiJhB2s)S=wHeyGjao&T9YBM0g;;e#lA34g2{m}iwJMx;IuA7 z=1?U)vKMy$W?Q0V5mrA>^(SL7`K1SEe`CBOR>VGx0%!>9;LFR~+lxRjF&(!zI7Clu@6Rk^_J-kTgfvH5@_>1^#!(^yAPN&<_hNXYPydmge?+jFRz(PDkE2 z`H!URMV>TU)AOI6$P!B~N>6lfc$;~0O|iiMBN5t*NCp%pS)4eriHvT)ELd?3HWO8{ zPDa9%&U`7G3yl3%p{A^5(u_dNr)YVxbGd7N^qrSVY$2LaKva}vV@E?n1J}09%XMqu zOrS-~d!0tR5Q#6X7@-T`vrj2UnV2hD{<2 za0o)e1U`z?ON(&%1uQ#DLTB%Bx++di@Kli;4hTEP939^vvv(4Bx$ncRjT4Q!mg@=9 z95x-!%Ew9>Bxc$V>rjw1)40;&(q(1|n_h2{hvSr1XQs}Xz2g*KjvQz`ZSH7C*Q}?^ ztO>pd7eeX$`6p;B$P3($i`#`MD459nKjo|9!=s}10@QB>Z6{4+aE>tCa3Js15bKD* zL`$$Rqc5*6d`27sY2aWyQw7$P1E_>PP)y z_;s?x5f`G4K3i(FRQ8CSo7QB$5_nU-mo4gR^nAxZ&~-u-e7vVmo)ir57&ee8DH4&S z?=@{cVr?zhl4a=+-#6(@>W7Xl$xmnm-XwiNWfX(546%NcA{o3GlIL(?ICy#2BkfRt z^FPe`5%e(r6v=~N9FkEB;7m$Ck`jQmYlJ)m87hQur5bp!LMRX#*M~;)-&cLoysBzefBYu8sbk zAc5!oSN`#@wi8q8flD8sKtVy^$u@+z$OgL0+%~thN)Aa-201kCx|yS$EK) z?NxRJo1WzVe0%jDU*jE*tnvQ+`|k$^uJn^P96jk>wgxw)oJ!4capsRy1Jrvn@(+~$ zHnaCUB7!Bc_=TJ6~Kk@=op&`)@&87JWLKWR0BDz?;%qtRKzvJ4np}81#q+JZLDCe2T6;(P+ViU)H~)B;oE2yC?QTXzu@i{7&Oa_b@&a1X$dUg=bx9un zIWYmB`!-Z)P5}W*#JFaEeUIuI7`TIED#^fzC5d0d-@e_3$3)Ayf(Ca>`r7r69);{c z6jWDNn}s1}LVShgm3wLYJ})+4{v?hKd5UpY1jOn+)EhjJG_ce_z57VpTT!<;A?by& zBfx(#a04LCyl=_4(lt2f3q#;58o7fLe-aNTQ4p@&LW3&^ZYEy~u763tM1^aKH$bNQ zww%(^qKDRS>dHKs>)w~67^9VV6pkRE+bg?4t?*Jep1%o(*|O{W?MkJGG#vs7df6>DIp!54+HnLz&8doN&9{4po8+!NBUMOa(3EZ`+G1IowmXj-z^7C0D zkDqjJSJzX$sh;1j*KiQ;8=%qsQ@;&~6MPg+!`F=rJa(nf$m>u@cu} zUL*eoBK;ntM$ocDl2~56SBQy3%I-jYd}m&v$WAE&r7=p_^N|;q5UvEQj7IrG{30(> zDVNOSEo*%?yWGK#4VWZMMGw0 zL_lB8PtP!-@z6xmas^q zp-H{KSHJFAY-|t0x#UG$Gh~9<5np-}0~3IJ&i~`n+N^rj(IHDsQ@hx|yi!ky>uUjH T4t`JsK`0(ok$ZXg{Ehz)XB)p= literal 0 HcmV?d00001 diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2ef79fbf17b329..6df9986b82e77c 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1861,7 +1861,7 @@ type_call(PyObject *self, PyObject *args, PyObject *kwds) PyObject * _PyType_NewManagedObject(PyTypeObject *type) { - assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(type->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(_PyType_IS_GC(type)); assert(type->tp_new == PyBaseObject_Type.tp_new); assert(type->tp_alloc == PyType_GenericAlloc); @@ -1870,11 +1870,6 @@ _PyType_NewManagedObject(PyTypeObject *type) if (obj == NULL) { return PyErr_NoMemory(); } - _PyObject_DictOrValuesPointer(obj)->dict = NULL; - if (_PyObject_InitInlineValues(obj, type)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -1888,9 +1883,13 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) * flag to indicate when that is safe) it does not seem worth the memory * savings. An example type that doesn't need the +1 is a subclass of * tuple. See GH-100659 and GH-81381. */ - const size_t size = _PyObject_VAR_SIZE(type, nitems+1); + size_t size = _PyObject_VAR_SIZE(type, nitems+1); const size_t presize = _PyType_PreHeaderSize(type); + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + assert(type->tp_itemsize == 0); + size += _PyInlineValuesSize(type); + } char *alloc = _PyObject_MallocWithType(type, size + presize); if (alloc == NULL) { return PyErr_NoMemory(); @@ -1911,6 +1910,9 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) else { _PyObject_InitVar((PyVarObject *)obj, type, nitems); } + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(obj, type); + } return obj; } @@ -2060,6 +2062,10 @@ subtype_clear(PyObject *self) if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { PyObject_ClearManagedDict(self); } + else { + assert((base->tp_flags & Py_TPFLAGS_INLINE_VALUES) == + (type->tp_flags & Py_TPFLAGS_INLINE_VALUES)); + } } else if (type->tp_dictoffset != base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -2210,14 +2216,7 @@ subtype_dealloc(PyObject *self) /* If we added a dict, DECREF it, or free inline values. */ if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(self); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - _PyObject_FreeInstanceAttributes(self); - } - else { - Py_XDECREF(_PyDictOrValues_GetDict(*dorv_ptr)); - } - dorv_ptr->values = NULL; + PyObject_ClearManagedDict(self); } else if (type->tp_dictoffset && !base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -3161,19 +3160,26 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) return func(descr, obj, value); } /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */ - dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return -1; - } if (value != NULL && !PyDict_Check(value)) { PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, " "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } - Py_XSETREF(*dictptr, Py_XNewRef(value)); + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + PyObject_ClearManagedDict(obj); + _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)Py_XNewRef(value); + } + else { + dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return -1; + } + Py_CLEAR(*dictptr); + *dictptr = Py_XNewRef(value); + } return 0; } @@ -5849,10 +5855,6 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (obj == NULL) { return NULL; } - if (_PyObject_InitializeDict(obj)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -6036,6 +6038,11 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* !same_slots_added(newbase, oldbase))) { goto differs; } + if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) != + ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES))) + { + goto differs; + } /* The above does not check for the preheader */ if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == ((newto->tp_flags & Py_TPFLAGS_PREHEADER))) @@ -6137,14 +6144,18 @@ object_set_class(PyObject *self, PyObject *value, void *closure) if (compatible_for_assignment(oldto, newto, "__class__")) { /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ - assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags & Py_TPFLAGS_PREHEADER)); - _PyObject_GetDictPtr(self); - if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self))) - { - /* Was unable to convert to dict */ - PyErr_NoMemory(); - return -1; + if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_ManagedDictPointer(self)->dict; + if (dict == NULL) { + dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(self); + if (dict == NULL) { + return -1; + } + _PyObject_ManagedDictPointer(self)->dict = dict; + } + if (_PyDict_DetachFromObject(dict, self)) { + return -1; + } } if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_INCREF(newto); @@ -7774,6 +7785,9 @@ type_ready_managed_dict(PyTypeObject *type) return -1; } } + if (type->tp_itemsize == 0 && type->tp_basicsize == sizeof(PyObject)) { + type->tp_flags |= Py_TPFLAGS_INLINE_VALUES; + } return 0; } @@ -7901,6 +7915,8 @@ PyType_Ready(PyTypeObject *type) /* Historically, all static types were immutable. See bpo-43908 */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + /* Static types must be immortal */ + _Py_SetImmortalUntracked((PyObject *)type); } int res; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bfb378c4a41500..ce208aac9c7953 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1897,14 +1897,12 @@ dummy_func( op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid); } split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; + attr = _PyObject_InlineValues(owner)->values[index]; DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); @@ -1947,16 +1945,15 @@ dummy_func( op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -2070,16 +2067,17 @@ dummy_func( DISPATCH_INLINED(new_frame); } - op(_GUARD_DORV_VALUES, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv)); + op(_GUARD_DORV_NO_DICT, (owner -- owner)) { + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict); + DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0); } op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); + assert(_PyObject_ManagedDictPointer(owner)->dict == NULL); + PyDictValues *values = _PyObject_InlineValues(owner); PyObject *old_value = values->values[index]; values->values[index] = value; if (old_value == NULL) { @@ -2094,7 +2092,7 @@ dummy_func( macro(STORE_ATTR_INSTANCE_VALUE) = unused/1 + _GUARD_TYPE_VERSION + - _GUARD_DORV_VALUES + + _GUARD_DORV_NO_DICT + _STORE_ATTR_INSTANCE_VALUE; inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { @@ -2102,9 +2100,8 @@ dummy_func( assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -2898,9 +2895,8 @@ dummy_func( } op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { @@ -2972,10 +2968,9 @@ dummy_func( unused/2 + _LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - op(_CHECK_ATTR_METHOD_LAZY_DICT, (owner -- owner)) { - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) { + char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL); } @@ -2993,7 +2988,7 @@ dummy_func( unused/1 + _GUARD_TYPE_VERSION + _CHECK_ATTR_METHOD_LAZY_DICT + - unused/2 + + unused/1 + _LOAD_ATTR_METHOD_LAZY_DICT; inst(INSTRUMENTED_CALL, (unused/3 -- )) { @@ -3294,6 +3289,7 @@ dummy_func( DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ce0dc235c54fcf..82f2171f1ede83 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1753,9 +1753,8 @@ PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET(); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET(); break; } @@ -1766,8 +1765,7 @@ (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; + attr = _PyObject_InlineValues(owner)->values[index]; if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); @@ -1784,8 +1782,7 @@ (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; + attr = _PyObject_InlineValues(owner)->values[index]; if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); @@ -1837,9 +1834,8 @@ PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET(); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; if (dict == NULL) JUMP_TO_JUMP_TARGET(); assert(PyDict_CheckExact((PyObject *)dict)); break; @@ -1852,8 +1848,8 @@ oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t hint = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -1967,12 +1963,13 @@ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _GUARD_DORV_VALUES: { + case _GUARD_DORV_NO_DICT: { PyObject *owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET(); + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_ManagedDictPointer(owner)->dict) JUMP_TO_JUMP_TARGET(); + if (_PyObject_InlineValues(owner)->valid == 0) JUMP_TO_JUMP_TARGET(); break; } @@ -1982,9 +1979,9 @@ owner = stack_pointer[-1]; value = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); + assert(_PyObject_ManagedDictPointer(owner)->dict == NULL); + PyDictValues *values = _PyObject_InlineValues(owner); PyObject *old_value = values->values[index]; values->values[index] = value; if (old_value == NULL) { @@ -2568,9 +2565,8 @@ case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { PyObject *owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET(); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET(); break; } @@ -2658,9 +2654,9 @@ case _CHECK_ATTR_METHOD_LAZY_DICT: { PyObject *owner; owner = stack_pointer[-1]; - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND(); + char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ if (dict != NULL) JUMP_TO_JUMP_TARGET(); break; diff --git a/Python/gc.c b/Python/gc.c index a37c1b144e57e9..a48738835fface 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2031,11 +2031,16 @@ gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) return op; } + PyObject * _PyObject_GC_New(PyTypeObject *tp) { size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize); + size_t size = _PyObject_SIZE(tp); + if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) { + size += _PyInlineValuesSize(tp); + } + PyObject *op = gc_alloc(tp, size, presize); if (op == NULL) { return NULL; } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 4524382e4f689f..7e4137a8e342b1 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1639,7 +1639,11 @@ PyObject * _PyObject_GC_New(PyTypeObject *tp) { size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize); + size_t size = _PyObject_SIZE(tp); + if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) { + size += _PyInlineValuesSize(tp); + } + PyObject *op = gc_alloc(tp, size, presize); if (op == NULL) { return NULL; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e8e2397b11cd48..6ee794a05b51d4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -870,6 +870,7 @@ DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; @@ -3680,15 +3681,13 @@ // _CHECK_MANAGED_OBJECT_HAS_VALUES { assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; + attr = _PyObject_InlineValues(owner)->values[index]; DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); @@ -3721,13 +3720,13 @@ } // _CHECK_ATTR_METHOD_LAZY_DICT { - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } - /* Skip 2 cache entries */ + /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3798,9 +3797,8 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -3914,9 +3912,8 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -4026,17 +4023,16 @@ // _CHECK_ATTR_WITH_HINT { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -5315,19 +5311,20 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - // _GUARD_DORV_VALUES + // _GUARD_DORV_NO_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict, STORE_ATTR); + DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; { uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); + assert(_PyObject_ManagedDictPointer(owner)->dict == NULL); + PyDictValues *values = _PyObject_InlineValues(owner); PyObject *old_value = values->values[index]; values->values[index] = value; if (old_value == NULL) { @@ -5389,9 +5386,8 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index df73cc091dea26..b4a1da8aec14af 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1104,7 +1104,7 @@ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ - case _GUARD_DORV_VALUES: { + case _GUARD_DORV_NO_DICT: { break; } diff --git a/Python/specialize.c b/Python/specialize.c index c1edf8842faf68..f1e32d05af7707 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -188,7 +188,7 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); - fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values); + fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs); fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs); @@ -197,7 +197,6 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); - fprintf(out, "Object dematerialize dict: %" PRIu64 "\n", stats->dict_dematerialized); fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); @@ -479,12 +478,11 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18 #define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19 #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20 - #define SPEC_FAIL_ATTR_SHADOWED 21 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 -#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25 + #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 @@ -558,6 +556,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 #define SPEC_FAIL_CALL_METACLASS 31 +#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -829,11 +828,7 @@ specialize_dict_access( return 0; } _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(*dorv) || - _PyObject_MakeInstanceAttributesFromDict(owner, dorv)) - { - // Virtual dictionary + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && _PyObject_InlineValues(owner)->valid) { PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); @@ -850,7 +845,8 @@ specialize_dict_access( instr->op.code = values_op; } else { - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; if (dict == NULL || !PyDict_CheckExact(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; @@ -1258,15 +1254,8 @@ PyObject *descr, DescriptorClassification kind, bool is_method) assert(descr != NULL); assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR)); - if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; - if (!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) - { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); - return 0; - } Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); if (index != DKIX_EMPTY) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); @@ -1282,10 +1271,16 @@ PyObject *descr, DescriptorClassification kind, bool is_method) instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; } else { - Py_ssize_t dictoffset = owner_cls->tp_dictoffset; - if (dictoffset < 0 || dictoffset > INT16_MAX) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); - return 0; + Py_ssize_t dictoffset; + if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + dictoffset = MANAGED_DICT_OFFSET; + } + else { + dictoffset = owner_cls->tp_dictoffset; + if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } } if (dictoffset == 0) { instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT; @@ -1296,8 +1291,12 @@ PyObject *descr, DescriptorClassification kind, bool is_method) SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } - assert(owner_cls->tp_dictoffset > 0); - assert(owner_cls->tp_dictoffset <= INT16_MAX); + /* Cache entries must be unsigned values, so we offset the + * dictoffset by MANAGED_DICT_OFFSET. + * We do the reverese offset in LOAD_ATTR_METHOD_LAZY_DICT */ + dictoffset -= MANAGED_DICT_OFFSET; + assert(((uint16_t)dictoffset) == dictoffset); + cache->dict_offset = (uint16_t)dictoffset; instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT; } else { @@ -1729,8 +1728,8 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp) SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN); return NULL; } - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT); + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES); return NULL; } if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index ddafcf99ca1e37..4261378d459107 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -359,11 +359,10 @@ def has_error_without_pop(op: parser.InstDef) -> bool: NON_ESCAPING_FUNCTIONS = ( "Py_INCREF", - "_PyDictOrValues_IsValues", - "_PyObject_DictOrValuesPointer", - "_PyDictOrValues_GetValues", + "_PyManagedDictPointer_IsValues", + "_PyObject_ManagedDictPointer", + "_PyObject_InlineValues", "_PyDictValues_AddToInsertionOrder", - "_PyObject_MakeInstanceAttributesFromDict", "Py_DECREF", "_Py_DECREF_SPECIALIZED", "DECREF_INPUTS_AND_REUSE_FLOAT", diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 656667ac93970c..74165acd831131 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -66,10 +66,12 @@ def _type_unsigned_short_ptr(): def _type_unsigned_int_ptr(): return gdb.lookup_type('unsigned int').pointer() - def _sizeof_void_p(): return gdb.lookup_type('void').pointer().sizeof +def _sizeof_pyobject(): + return gdb.lookup_type('PyObject').sizeof + def _managed_dict_offset(): # See pycore_object.h pyobj = gdb.lookup_type("PyObject") @@ -79,6 +81,7 @@ def _managed_dict_offset(): return -3 * _sizeof_void_p() +Py_TPFLAGS_INLINE_VALUES = (1 << 2) Py_TPFLAGS_MANAGED_DICT = (1 << 4) Py_TPFLAGS_HEAPTYPE = (1 << 9) Py_TPFLAGS_LONG_SUBCLASS = (1 << 24) @@ -493,11 +496,12 @@ def get_keys_values(self): has_values = int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT if not has_values: return None - ptr = self._gdbval.cast(_type_char_ptr()) + _managed_dict_offset() - char_ptr = ptr.cast(_type_char_ptr().pointer()).dereference() - if (int(char_ptr) & 1) == 0: + obj_ptr = self._gdbval.cast(_type_char_ptr()) + dict_ptr_ptr = obj_ptr + _managed_dict_offset() + dict_ptr = dict_ptr_ptr.cast(_type_char_ptr().pointer()).dereference() + if int(dict_ptr): return None - char_ptr += 1 + char_ptr = obj_ptr + _sizeof_pyobject() values_ptr = char_ptr.cast(gdb.lookup_type("PyDictValues").pointer()) values = values_ptr['values'] return PyKeysValuesPair(self.get_cached_keys(), values) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 8dc590b4b89a88..f7ed98ff6045ab 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -394,7 +394,7 @@ def get_call_stats(self) -> dict[str, int]: return result def get_object_stats(self) -> dict[str, tuple[int, int]]: - total_materializations = self._data.get("Object new values", 0) + total_materializations = self._data.get("Object inline values", 0) total_allocations = self._data.get("Object allocations", 0) + self._data.get( "Object allocations from freelist", 0 ) @@ -1094,8 +1094,7 @@ def calc_object_stats_table(stats: Stats) -> Rows: Below, "allocations" means "allocations that are not from a freelist". Total allocations = "Allocations from freelist" + "Allocations". - "New values" is the number of values arrays created for objects with - managed dicts. + "Inline values" is the number of values arrays inlined into objects. The cache hit/miss numbers are for the MRO cache, split into dunder and other names. From e3b6f287fc5d195859c29661145873c638c6dc83 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 2 Apr 2024 13:29:39 +0200 Subject: [PATCH 047/143] gh-113317: Argument Clinic: Add libclinic.return_converters (#117451) Move the following converter classes to libclinic.return_converters: * CReturnConverter * CReturnConverterAutoRegister * Py_ssize_t_return_converter * bool_return_converter * double_return_converter * float_return_converter * int_return_converter * long_return_converter * size_t_return_converter * unsigned_int_return_converter * unsigned_long_return_converter Move also the add_c_return_converter() function there. --- Tools/clinic/clinic.py | 183 +------------------- Tools/clinic/libclinic/function.py | 3 +- Tools/clinic/libclinic/return_converters.py | 173 ++++++++++++++++++ 3 files changed, 178 insertions(+), 181 deletions(-) create mode 100644 Tools/clinic/libclinic/return_converters.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a4e004d5b124d1..97b1f46a13411b 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -60,6 +60,9 @@ from libclinic.converters import ( self_converter, defining_class_converter, object_converter, buffer, robuffer, rwbuffer, correct_name_for_self) +from libclinic.return_converters import ( + CReturnConverter, return_converters, + int_return_converter, ReturnConverterType) # TODO: @@ -2088,186 +2091,6 @@ def parse(self, block: Block) -> None: """.strip().split()) -ReturnConverterType = Callable[..., "CReturnConverter"] - - -# maps strings to callables. -# these callables must be of the form: -# def foo(*, ...) -# The callable may have any number of keyword-only parameters. -# The callable must return a CReturnConverter object. -# The callable should not call builtins.print. -ReturnConverterDict = dict[str, ReturnConverterType] -return_converters: ReturnConverterDict = {} - - -def add_c_return_converter( - f: ReturnConverterType, - name: str | None = None -) -> ReturnConverterType: - if not name: - name = f.__name__ - if not name.endswith('_return_converter'): - return f - name = name.removesuffix('_return_converter') - return_converters[name] = f - return f - - -class CReturnConverterAutoRegister(type): - def __init__( - cls: ReturnConverterType, - name: str, - bases: tuple[type[object], ...], - classdict: dict[str, Any] - ) -> None: - add_c_return_converter(cls) - - -class CReturnConverter(metaclass=CReturnConverterAutoRegister): - - # The C type to use for this variable. - # 'type' should be a Python string specifying the type, e.g. "int". - # If this is a pointer type, the type string should end with ' *'. - type = 'PyObject *' - - # The Python default value for this parameter, as a Python value. - # Or the magic value "unspecified" if there is no default. - default: object = None - - def __init__( - self, - *, - py_default: str | None = None, - **kwargs: Any - ) -> None: - self.py_default = py_default - try: - self.return_converter_init(**kwargs) - except TypeError as e: - s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) - sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) - - def return_converter_init(self) -> None: ... - - def declare(self, data: CRenderData) -> None: - line: list[str] = [] - add = line.append - add(self.type) - if not self.type.endswith('*'): - add(' ') - add(data.converter_retval + ';') - data.declarations.append(''.join(line)) - data.return_value = data.converter_retval - - def err_occurred_if( - self, - expr: str, - data: CRenderData - ) -> None: - line = f'if (({expr}) && PyErr_Occurred()) {{\n goto exit;\n}}\n' - data.return_conversion.append(line) - - def err_occurred_if_null_pointer( - self, - variable: str, - data: CRenderData - ) -> None: - line = f'if ({variable} == NULL) {{\n goto exit;\n}}\n' - data.return_conversion.append(line) - - def render( - self, - function: Function, - data: CRenderData - ) -> None: ... - - -add_c_return_converter(CReturnConverter, 'object') - - -class bool_return_converter(CReturnConverter): - type = 'int' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: - self.declare(data) - self.err_occurred_if(f"{data.converter_retval} == -1", data) - data.return_conversion.append( - f'return_value = PyBool_FromLong((long){data.converter_retval});\n' - ) - - -class long_return_converter(CReturnConverter): - type = 'long' - conversion_fn = 'PyLong_FromLong' - cast = '' - unsigned_cast = '' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: - self.declare(data) - self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) - data.return_conversion.append( - f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' - ) - - -class int_return_converter(long_return_converter): - type = 'int' - cast = '(long)' - - -class unsigned_long_return_converter(long_return_converter): - type = 'unsigned long' - conversion_fn = 'PyLong_FromUnsignedLong' - unsigned_cast = '(unsigned long)' - - -class unsigned_int_return_converter(unsigned_long_return_converter): - type = 'unsigned int' - cast = '(unsigned long)' - unsigned_cast = '(unsigned int)' - - -class Py_ssize_t_return_converter(long_return_converter): - type = 'Py_ssize_t' - conversion_fn = 'PyLong_FromSsize_t' - - -class size_t_return_converter(long_return_converter): - type = 'size_t' - conversion_fn = 'PyLong_FromSize_t' - unsigned_cast = '(size_t)' - - -class double_return_converter(CReturnConverter): - type = 'double' - cast = '' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: - self.declare(data) - self.err_occurred_if(f"{data.converter_retval} == -1.0", data) - data.return_conversion.append( - f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' - ) - - -class float_return_converter(double_return_converter): - type = 'float' - cast = '(double)' - - def eval_ast_expr( node: ast.expr, *, diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index b0dd08446e802d..1bfaad00cd0f08 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -6,9 +6,10 @@ import inspect from typing import Final, Any, TYPE_CHECKING if TYPE_CHECKING: - from clinic import Clinic, CReturnConverter + from clinic import Clinic from libclinic.converter import CConverter from libclinic.converters import self_converter + from libclinic.return_converters import CReturnConverter from libclinic import VersionTuple, unspecified diff --git a/Tools/clinic/libclinic/return_converters.py b/Tools/clinic/libclinic/return_converters.py new file mode 100644 index 00000000000000..7bdd257cfa3443 --- /dev/null +++ b/Tools/clinic/libclinic/return_converters.py @@ -0,0 +1,173 @@ +import sys +from collections.abc import Callable +from libclinic.crenderdata import CRenderData +from libclinic.function import Function +from typing import Any + + +ReturnConverterType = Callable[..., "CReturnConverter"] + + +# maps strings to callables. +# these callables must be of the form: +# def foo(*, ...) +# The callable may have any number of keyword-only parameters. +# The callable must return a CReturnConverter object. +# The callable should not call builtins.print. +ReturnConverterDict = dict[str, ReturnConverterType] +return_converters: ReturnConverterDict = {} + + +def add_c_return_converter( + f: ReturnConverterType, + name: str | None = None +) -> ReturnConverterType: + if not name: + name = f.__name__ + if not name.endswith('_return_converter'): + return f + name = name.removesuffix('_return_converter') + return_converters[name] = f + return f + + +class CReturnConverterAutoRegister(type): + def __init__( + cls: ReturnConverterType, + name: str, + bases: tuple[type[object], ...], + classdict: dict[str, Any] + ) -> None: + add_c_return_converter(cls) + + +class CReturnConverter(metaclass=CReturnConverterAutoRegister): + + # The C type to use for this variable. + # 'type' should be a Python string specifying the type, e.g. "int". + # If this is a pointer type, the type string should end with ' *'. + type = 'PyObject *' + + # The Python default value for this parameter, as a Python value. + # Or the magic value "unspecified" if there is no default. + default: object = None + + def __init__( + self, + *, + py_default: str | None = None, + **kwargs: Any + ) -> None: + self.py_default = py_default + try: + self.return_converter_init(**kwargs) + except TypeError as e: + s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) + sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) + + def return_converter_init(self) -> None: ... + + def declare(self, data: CRenderData) -> None: + line: list[str] = [] + add = line.append + add(self.type) + if not self.type.endswith('*'): + add(' ') + add(data.converter_retval + ';') + data.declarations.append(''.join(line)) + data.return_value = data.converter_retval + + def err_occurred_if( + self, + expr: str, + data: CRenderData + ) -> None: + line = f'if (({expr}) && PyErr_Occurred()) {{\n goto exit;\n}}\n' + data.return_conversion.append(line) + + def err_occurred_if_null_pointer( + self, + variable: str, + data: CRenderData + ) -> None: + line = f'if ({variable} == NULL) {{\n goto exit;\n}}\n' + data.return_conversion.append(line) + + def render( + self, + function: Function, + data: CRenderData + ) -> None: ... + + +add_c_return_converter(CReturnConverter, 'object') + + +class bool_return_converter(CReturnConverter): + type = 'int' + + def render(self, function: Function, data: CRenderData) -> None: + self.declare(data) + self.err_occurred_if(f"{data.converter_retval} == -1", data) + data.return_conversion.append( + f'return_value = PyBool_FromLong((long){data.converter_retval});\n' + ) + + +class long_return_converter(CReturnConverter): + type = 'long' + conversion_fn = 'PyLong_FromLong' + cast = '' + unsigned_cast = '' + + def render(self, function: Function, data: CRenderData) -> None: + self.declare(data) + self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) + data.return_conversion.append( + f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' + ) + + +class int_return_converter(long_return_converter): + type = 'int' + cast = '(long)' + + +class unsigned_long_return_converter(long_return_converter): + type = 'unsigned long' + conversion_fn = 'PyLong_FromUnsignedLong' + unsigned_cast = '(unsigned long)' + + +class unsigned_int_return_converter(unsigned_long_return_converter): + type = 'unsigned int' + cast = '(unsigned long)' + unsigned_cast = '(unsigned int)' + + +class Py_ssize_t_return_converter(long_return_converter): + type = 'Py_ssize_t' + conversion_fn = 'PyLong_FromSsize_t' + + +class size_t_return_converter(long_return_converter): + type = 'size_t' + conversion_fn = 'PyLong_FromSize_t' + unsigned_cast = '(size_t)' + + +class double_return_converter(CReturnConverter): + type = 'double' + cast = '' + + def render(self, function: Function, data: CRenderData) -> None: + self.declare(data) + self.err_occurred_if(f"{data.converter_retval} == -1.0", data) + data.return_conversion.append( + f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' + ) + + +class float_return_converter(double_return_converter): + type = 'float' + cast = '(double)' From 52f5b7f9e05fc4a25e385c046e0b091641674556 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 2 Apr 2024 21:10:24 +0800 Subject: [PATCH 048/143] gh-115538: Use pathlib to compare prefixes in test_venv (GH-117076) --- Lib/test/test_venv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 63cc4b743862bc..f410ce7198dc86 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -273,7 +273,8 @@ def test_prefixes(self): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), prefix) + self.assertEqual(pathlib.Path(out.strip().decode()), + pathlib.Path(expected), prefix) @requireVenvCreate def test_sysconfig(self): From e569f9132b5bdc1c103116a020e19e3ccc20cf34 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 2 Apr 2024 16:08:16 +0200 Subject: [PATCH 049/143] gh-117074: Update Traversable.joinpath docs to the 3.11+ protocol (GH-117113) --- Doc/library/importlib.resources.abc.rst | 26 +++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index c508b6ba965cc0..c25c48530722e6 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -109,13 +109,35 @@ Return True if self is a file. - .. abstractmethod:: joinpath(child) + .. abstractmethod:: joinpath(*pathsegments) - Return Traversable child in self. + Traverse directories according to *pathsegments* and return + the result as :class:`!Traversable`. + + Each *pathsegments* argument may contain multiple names separated by + forward slashes (``/``, ``posixpath.sep`` ). + For example, the following are equivalent:: + + files.joinpath('subdir', 'subsuddir', 'file.txt') + files.joinpath('subdir/subsuddir/file.txt') + + Note that some :class:`!Traversable` implementations + might not be updated to the latest version of the protocol. + For compatibility with such implementations, provide a single argument + without path separators to each call to ``joinpath``. For example:: + + files.joinpath('subdir').joinpath('subsubdir').joinpath('file.txt') + + .. versionchanged:: 3.11 + + ``joinpath`` accepts multiple *pathsegments*, and these segments + may contain forward slashes as path separators. + Previously, only a single *child* argument was accepted. .. abstractmethod:: __truediv__(child) Return Traversable child in self. + Equivalent to ``joinpath(child)``. .. abstractmethod:: open(mode='r', *args, **kwargs) From 954d616b4c8cd091214aa3b8ea886bcf9067243a Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 2 Apr 2024 10:44:26 -0400 Subject: [PATCH 050/143] gh-117440: Make `syslog` thread-safe in free-threaded builds (#117441) Use critical sections to protect access to the syslog module. --- Modules/clinic/syslogmodule.c.h | 15 +++++++++++++-- Modules/syslogmodule.c | 9 ++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Modules/clinic/syslogmodule.c.h b/Modules/clinic/syslogmodule.c.h index 58b0ea57b065b3..77cf24ea5ba9ca 100644 --- a/Modules/clinic/syslogmodule.c.h +++ b/Modules/clinic/syslogmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(syslog_openlog__doc__, @@ -88,7 +89,9 @@ syslog_openlog(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(module); return_value = syslog_openlog_impl(module, ident, logopt, facility); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -129,7 +132,9 @@ syslog_syslog(PyObject *module, PyObject *args) PyErr_SetString(PyExc_TypeError, "syslog.syslog requires 1 to 2 arguments"); goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = syslog_syslog_impl(module, group_left_1, priority, message); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -150,7 +155,13 @@ syslog_closelog_impl(PyObject *module); static PyObject * syslog_closelog(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return syslog_closelog_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = syslog_closelog_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(syslog_setlogmask__doc__, @@ -251,4 +262,4 @@ syslog_LOG_UPTO(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=86ca2fd84b2da98e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8d25899bd31969d3 input=a9049054013a1b77]*/ diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 62c7816f891ee2..cb3f2b03990cb8 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -132,6 +132,7 @@ syslog_get_argv(void) /*[clinic input] +@critical_section syslog.openlog ident: unicode = NULL @@ -144,7 +145,7 @@ Set logging options of subsequent syslog() calls. static PyObject * syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, long facility) -/*[clinic end generated code: output=5476c12829b6eb75 input=8a987a96a586eee7]*/ +/*[clinic end generated code: output=5476c12829b6eb75 input=ee700b8786f81c23]*/ { // Since the sys.openlog changes the process level state of syslog library, // this operation is only allowed for the main interpreter. @@ -189,6 +190,7 @@ syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, /*[clinic input] +@critical_section syslog.syslog [ @@ -205,7 +207,7 @@ Send the string message to the system logger. static PyObject * syslog_syslog_impl(PyObject *module, int group_left_1, int priority, const char *message) -/*[clinic end generated code: output=c3dbc73445a0e078 input=ac83d92b12ea3d4e]*/ +/*[clinic end generated code: output=c3dbc73445a0e078 input=6588ddb0b113af8e]*/ { if (PySys_Audit("syslog.syslog", "is", priority, message) < 0) { return NULL; @@ -243,6 +245,7 @@ syslog_syslog_impl(PyObject *module, int group_left_1, int priority, /*[clinic input] +@critical_section syslog.closelog Reset the syslog module values and call the system library closelog(). @@ -250,7 +253,7 @@ Reset the syslog module values and call the system library closelog(). static PyObject * syslog_closelog_impl(PyObject *module) -/*[clinic end generated code: output=97890a80a24b1b84 input=fb77a54d447acf07]*/ +/*[clinic end generated code: output=97890a80a24b1b84 input=167f489868bd5a72]*/ { // Since the sys.closelog changes the process level state of syslog library, // this operation is only allowed for the main interpreter. From 027fa2eccf39ddccdf7b416d16601277a7112054 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 2 Apr 2024 10:45:00 -0400 Subject: [PATCH 051/143] gh-112087: Make `list.extend(dict)` behave atomically (#117438) Add a special case for `list.extend(dict)` and `list(dict)` so that those patterns behave atomically with respect to modifications to the list or dictionary. This is required by multiprocessing, which assumes that `list(_finalizer_registry)` is atomic. --- Objects/listobject.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Objects/listobject.c b/Objects/listobject.c index 470ad8eb8135db..472c471d9968a4 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1374,6 +1374,11 @@ _list_extend(PyListObject *self, PyObject *iterable) res = list_extend_set(self, (PySetObject *)iterable); Py_END_CRITICAL_SECTION2(); } + else if (PyDict_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_dict(self, (PyDictObject *)iterable, 0 /*keys*/); + Py_END_CRITICAL_SECTION2(); + } else if (Py_IS_TYPE(iterable, &PyDictKeys_Type)) { PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; Py_BEGIN_CRITICAL_SECTION2(self, dict); From 8eda146e87d5531c9d2bc62fd1fea3bd3163f9b1 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 2 Apr 2024 11:25:48 -0700 Subject: [PATCH 052/143] Fix successor opcode name printing in Tier 2 DEOPT debug message (#117471) --- Python/ceval.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index 1b13eb1702355f..f3b73165e9f28b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1078,7 +1078,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int printf("DEOPT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(" -> %s]\n", - _PyOpcode_OpName[frame->instr_ptr->op.code]); + _PyOpcode_OpName[next_instr->op.code]); } #endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); From cae4cdd07ddfcd8bcc05d683bac53815391c9907 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 2 Apr 2024 22:32:35 +0200 Subject: [PATCH 053/143] gh-117349: Micro-optimize a few `os.path` functions (#117350) Co-authored-by: Alex Waygood Co-authored-by: Barney Gale Co-authored-by: Pieter Eendebak --- Lib/ntpath.py | 29 ++++++------- Lib/posixpath.py | 43 +++++++++---------- Misc/ACKS | 1 + ...-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst | 1 + 4 files changed, 35 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index ecfc7d48dbb192..0650f14f89f10b 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -102,11 +102,11 @@ def join(path, *paths): if isinstance(path, bytes): sep = b'\\' seps = b'\\/' - colon = b':' + colon_seps = b':\\/' else: sep = '\\' seps = '\\/' - colon = ':' + colon_seps = ':\\/' try: if not paths: path[:0] + sep #23780: Ensure compatible data type even if p is null. @@ -135,7 +135,7 @@ def join(path, *paths): result_path = result_path + p_path ## add separator between UNC and non-absolute path if (result_path and not result_root and - result_drive and result_drive[-1:] not in colon + seps): + result_drive and result_drive[-1] not in colon_seps): return result_drive + sep + result_path return result_drive + result_root + result_path except (TypeError, AttributeError, BytesWarning): @@ -279,7 +279,7 @@ def isjunction(path): st = os.lstat(path) except (OSError, ValueError, AttributeError): return False - return bool(st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT) + return st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT else: # Use genericpath.isjunction as imported above pass @@ -340,8 +340,8 @@ def isreserved(path): def _isreservedname(name): """Return true if the filename is reserved by the system.""" # Trailing dots and spaces are reserved. - if name.endswith(('.', ' ')) and name not in ('.', '..'): - return True + if name[-1:] in ('.', ' '): + return name not in ('.', '..') # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved. # ASCII control characters (0-31) are reserved. # Colon is reserved for file streams (e.g. "name:stream[:type]"). @@ -350,9 +350,7 @@ def _isreservedname(name): # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules # are complex and vary across Windows versions. On the side of # caution, return True for names that may not be reserved. - if name.partition('.')[0].rstrip(' ').upper() in _reserved_names: - return True - return False + return name.partition('.')[0].rstrip(' ').upper() in _reserved_names # Expand paths beginning with '~' or '~user'. @@ -381,13 +379,10 @@ def expanduser(path): if 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] - elif not 'HOMEPATH' in os.environ: + elif 'HOMEPATH' not in os.environ: return path else: - try: - drive = os.environ['HOMEDRIVE'] - except KeyError: - drive = '' + drive = os.environ.get('HOMEDRIVE', '') userhome = join(drive, os.environ['HOMEPATH']) if i != 1: #~user @@ -727,7 +722,8 @@ def realpath(path, *, strict=False): new_unc_prefix = b'\\\\' cwd = os.getcwdb() # bpo-38081: Special case for realpath(b'nul') - if normcase(path) == normcase(os.fsencode(devnull)): + devnull = b'nul' + if normcase(path) == devnull: return b'\\\\.\\NUL' else: prefix = '\\\\?\\' @@ -735,7 +731,8 @@ def realpath(path, *, strict=False): new_unc_prefix = '\\\\' cwd = os.getcwd() # bpo-38081: Special case for realpath('nul') - if normcase(path) == normcase(devnull): + devnull = 'nul' + if normcase(path) == devnull: return '\\\\.\\NUL' had_prefix = path.startswith(prefix) if not had_prefix and not isabs(path): diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 4fc02be69bd6e1..76ee721bfb5e33 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -213,15 +213,8 @@ def ismount(path): except (OSError, ValueError): return False - dev1 = s1.st_dev - dev2 = s2.st_dev - if dev1 != dev2: - return True # path/.. on a different device as path - ino1 = s1.st_ino - ino2 = s2.st_ino - if ino1 == ino2: - return True # path/.. is the same i-node as path - return False + # path/.. on a different device as path or the same i-node as path + return s1.st_dev != s2.st_dev or s1.st_ino == s2.st_ino # Expand paths beginning with '~' or '~user'. @@ -270,7 +263,7 @@ def expanduser(path): return path name = path[1:i] if isinstance(name, bytes): - name = str(name, 'ASCII') + name = name.decode('ascii') try: pwent = pwd.getpwnam(name) except KeyError: @@ -359,21 +352,19 @@ def normpath(path): path = os.fspath(path) if isinstance(path, bytes): sep = b'/' - empty = b'' dot = b'.' dotdot = b'..' else: sep = '/' - empty = '' dot = '.' dotdot = '..' - if path == empty: + if not path: return dot _, initial_slashes, path = splitroot(path) comps = path.split(sep) new_comps = [] for comp in comps: - if comp in (empty, dot): + if not comp or comp == dot: continue if (comp != dotdot or (not initial_slashes and not new_comps) or (new_comps and new_comps[-1] == dotdot)): @@ -396,12 +387,12 @@ def normpath(path): def abspath(path): """Return an absolute path.""" path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) + if isinstance(path, bytes): + if not path.startswith(b'/'): + path = join(os.getcwdb(), path) + else: + if not path.startswith('/'): + path = join(os.getcwd(), path) return normpath(path) @@ -417,6 +408,7 @@ def realpath(filename, *, strict=False): # Join two paths, normalizing and eliminating any symbolic links # encountered in the second path. +# Two leading slashes are replaced by a single slash. def _joinrealpath(path, rest, strict, seen): if isinstance(path, bytes): sep = b'/' @@ -427,7 +419,7 @@ def _joinrealpath(path, rest, strict, seen): curdir = '.' pardir = '..' - if isabs(rest): + if rest.startswith(sep): rest = rest[1:] path = sep @@ -439,10 +431,15 @@ def _joinrealpath(path, rest, strict, seen): if name == pardir: # parent dir if path: - path, name = split(path) + parent, name = split(path) if name == pardir: - path = join(path, pardir, pardir) + # ../.. + path = join(path, pardir) + else: + # foo/bar/.. -> foo + path = parent else: + # .. path = pardir continue newpath = join(path, name) diff --git a/Misc/ACKS b/Misc/ACKS index 03e458d170d2b8..fe014a364dd82d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -191,6 +191,7 @@ Finn Bock Paul Boddie Matthew Boedicker Robin Boerdijk +Wannes Boeykens Andra Bogildea Matt Bogosian Nikolay Bogoychev diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst new file mode 100644 index 00000000000000..7a7bc689002017 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst @@ -0,0 +1 @@ +Optimise several functions in :mod:`os.path`. From f341d6017dd4e80509b69b5a9e2625b71b70f205 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 14:35:52 -0600 Subject: [PATCH 054/143] gh-76785: Add PyInterpreterConfig Helpers (gh-117170) These helpers make it easier to customize and inspect the config used to initialize interpreters. This is especially valuable in our tests. I found inspiration from the PyConfig API for the PyInterpreterConfig dict conversion stuff. As part of this PR I've also added a bunch of tests. --- Include/internal/pycore_pylifecycle.h | 16 ++ Lib/test/support/__init__.py | 15 +- Lib/test/test_capi/test_misc.py | 251 +++++++++++++++++++++++ Lib/test/test_import/__init__.py | 12 +- Makefile.pre.in | 5 + Modules/_testinternalcapi.c | 206 +++++++++++++------ PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/config_common.h | 36 ++++ Python/initconfig.c | 25 +-- Python/interpconfig.c | 266 +++++++++++++++++++++++++ 13 files changed, 754 insertions(+), 86 deletions(-) create mode 100644 Python/config_common.h create mode 100644 Python/interpconfig.c diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index c675098685764c..47ff0806574ac0 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -116,6 +116,22 @@ PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); // Export for special main.c string compiling with source tracebacks int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + +/* interpreter config */ + +// Export for _testinternalcapi shared extension +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromState( + PyInterpreterConfig *, + PyInterpreterState *); +PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromDict( + PyInterpreterConfig *, + PyObject *); +PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict( + PyInterpreterConfig *, + PyObject *); + + #ifdef __cplusplus } #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 92e3174407f133..9640d5d831b874 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1734,8 +1734,19 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): raise unittest.SkipTest("requires _testinternalcapi") if own_gil is not None: assert 'gil' not in config, (own_gil, config) - config['gil'] = 2 if own_gil else 1 - return _testinternalcapi.run_in_subinterp_with_config(code, **config) + config['gil'] = 'own' if own_gil else 'shared' + else: + gil = config['gil'] + if gil == 0: + config['gil'] = 'default' + elif gil == 1: + config['gil'] = 'shared' + elif gil == 2: + config['gil'] = 'own' + else: + raise NotImplementedError(gil) + config = types.SimpleNamespace(**config) + return _testinternalcapi.run_in_subinterp_with_config(code, config) def _check_tracemalloc(): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 55a1ab6d6d9359..34311afc93fc29 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2204,6 +2204,257 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +class InterpreterConfigTests(unittest.TestCase): + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=False, + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + def iter_all_configs(self): + for use_main_obmalloc in (True, False): + for allow_fork in (True, False): + for allow_exec in (True, False): + for allow_threads in (True, False): + for allow_daemon in (True, False): + for checkext in (True, False): + for gil in ('shared', 'own', 'default'): + yield types.SimpleNamespace( + use_main_obmalloc=use_main_obmalloc, + allow_fork=allow_fork, + allow_exec=allow_exec, + allow_threads=allow_threads, + allow_daemon_threads=allow_daemon, + check_multi_interp_extensions=checkext, + gil=gil, + ) + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def test_predefined_config(self): + def check(name, expected): + expected = self.supported[expected] + args = (name,) if name else () + + config1 = _testinternalcapi.new_interp_config(*args) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _testinternalcapi.new_interp_config(*args) + self.assert_ns_equal(config2, expected) + self.assertIsNot(config2, expected) + self.assertIsNot(config2, config1) + + with self.subTest('default'): + check(None, 'isolated') + + for name in self.supported: + with self.subTest(name): + check(name, name) + + def test_update_from_dict(self): + for name, vanilla in self.supported.items(): + with self.subTest(f'noop ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _testinternalcapi.new_interp_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'change all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in self.gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _testinternalcapi.new_interp_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in self.gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _testinternalcapi.new_interp_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('unsupported field'): + for name in self.supported: + with self.assertRaises(ValueError): + _testinternalcapi.new_interp_config(name, spam=True) + + # Bad values for bool fields. + for field, value in vars(self.supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'unsupported value ({field}={value!r})'): + with self.assertRaises(TypeError): + _testinternalcapi.new_interp_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'unsupported value(gil={value!r})'): + with self.assertRaises(TypeError): + _testinternalcapi.new_interp_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'unsupported value (gil={value!r})'): + with self.assertRaises(ValueError): + _testinternalcapi.new_interp_config(gil=value) + + @requires_subinterpreters + def test_interp_init(self): + questionable = [ + # strange + dict( + allow_fork=True, + allow_exec=False, + ), + dict( + gil='shared', + use_main_obmalloc=False, + ), + # risky + dict( + allow_fork=True, + allow_threads=True, + ), + # ought to be invalid? + dict( + allow_threads=False, + allow_daemon_threads=True, + ), + dict( + gil='own', + use_main_obmalloc=True, + ), + ] + invalid = [ + dict( + use_main_obmalloc=False, + check_multi_interp_extensions=False + ), + ] + def match(config, override_cases): + ns = vars(config) + for overrides in override_cases: + if dict(ns, **overrides) == ns: + return True + return False + + def check(config): + script = 'pass' + rc = _testinternalcapi.run_in_subinterp_with_config(script, config) + self.assertEqual(rc, 0) + + for config in self.iter_all_configs(): + if config.gil == 'default': + continue + if match(config, invalid): + with self.subTest(f'invalid: {config}'): + with self.assertRaises(RuntimeError): + check(config) + elif match(config, questionable): + with self.subTest(f'questionable: {config}'): + check(config) + else: + with self.subTest(f'valid: {config}'): + check(config) + + @requires_subinterpreters + def test_get_config(self): + @contextlib.contextmanager + def new_interp(config): + interpid = _testinternalcapi.new_interpreter(config) + try: + yield interpid + finally: + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + with self.subTest('main'): + expected = _testinternalcapi.new_interp_config('legacy') + expected.gil = 'own' + interpid = _interpreters.get_main() + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _testinternalcapi.new_interp_config('isolated') + with new_interp('isolated') as interpid: + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _testinternalcapi.new_interp_config('legacy') + with new_interp('legacy') as interpid: + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _testinternalcapi.new_interp_config( + 'empty', + use_main_obmalloc=True, + gil='shared', + ) + with new_interp(orig) as interpid: + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, orig) + + @requires_subinterpreters class InterpreterIDTests(unittest.TestCase): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 4deed7f3ba2522..81ec700d9755ce 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1823,15 +1823,19 @@ def check_compatible_fresh(self, name, *, strict=False, isolated=False): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=strict, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert ( {name!r} in sys.builtin_module_names or {name!r} not in sys.modules ), repr({name!r}) + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) @@ -1847,12 +1851,16 @@ def check_incompatible_fresh(self, name, *, isolated=False): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=True, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert {name!r} not in sys.modules, {name!r} + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) diff --git a/Makefile.pre.in b/Makefile.pre.in index f5c2af0696ac33..2a22a1e95a39a2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -440,6 +440,7 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/interpconfig.o \ Python/instrumentation.o \ Python/intrinsics.o \ Python/jit.o \ @@ -1687,6 +1688,10 @@ Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $ Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h +Python/initconfig.o: $(srcdir)/Python/initconfig.c $(srcdir)/Python/config_common.h + +Python/interpconfig.o: $(srcdir)/Python/interpconfig.c $(srcdir)/Python/config_common.h + Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile $(CC) -c $(PY_CORE_CFLAGS) \ -DSOABI='"$(SOABI)"' \ diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index d6d50e75b612df..56761d1a896d2a 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,10 +23,12 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyThreadState_GET() #include "clinic/_testinternalcapi.c.h" @@ -1355,83 +1357,153 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } -/* To run some code in a sub-interpreter. */ -static PyObject * -run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) +static int +init_named_interp_config(PyInterpreterConfig *config, const char *name) { - const char *code; - int use_main_obmalloc = -1; - int allow_fork = -1; - int allow_exec = -1; - int allow_threads = -1; - int allow_daemon_threads = -1; - int check_multi_interp_extensions = -1; - int gil = -1; - int r; - PyThreadState *substate, *mainstate; - /* only initialise 'cflags.cf_flags' to test backwards compatibility */ - PyCompilerFlags cflags = {0}; + if (name == NULL) { + name = "isolated"; + } - static char *kwlist[] = {"code", - "use_main_obmalloc", - "allow_fork", - "allow_exec", - "allow_threads", - "allow_daemon_threads", - "check_multi_interp_extensions", - "gil", - NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$ppppppi:run_in_subinterp_with_config", kwlist, - &code, &use_main_obmalloc, - &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions, - &gil)) { + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + +static PyObject * +new_interp_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:new_config", &name)) { return NULL; } - if (use_main_obmalloc < 0) { - PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc"); + PyObject *overrides = kwds; + + if (name == NULL) { + name = "isolated"; + } + + PyInterpreterConfig config; + if (init_named_interp_config(&config, name) < 0) { return NULL; } - if (allow_fork < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_fork"); + + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { return NULL; } - if (allow_exec < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_exec"); + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static PyObject * +get_interp_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:get_config", kwlist, &idobj)) + { return NULL; } - if (allow_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_threads"); + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { return NULL; } - if (gil < 0) { - PyErr_SetString(PyExc_ValueError, "missing gil"); + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { return NULL; } - if (allow_daemon_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static int +interp_config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_interp_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + if (init_named_interp_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + + +/* To run some code in a sub-interpreter. */ +static PyObject * +run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *code; + PyObject *configobj; + static char *kwlist[] = {"code", "config", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "sO:run_in_subinterp_with_config", kwlist, + &code, &configobj)) + { return NULL; } - if (check_multi_interp_extensions < 0) { - PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + + PyInterpreterConfig config; + if (interp_config_from_object(configobj, &config) < 0) { return NULL; } - mainstate = PyThreadState_Get(); + PyThreadState *mainstate = PyThreadState_Get(); PyThreadState_Swap(NULL); - const PyInterpreterConfig config = { - .use_main_obmalloc = use_main_obmalloc, - .allow_fork = allow_fork, - .allow_exec = allow_exec, - .allow_threads = allow_threads, - .allow_daemon_threads = allow_daemon_threads, - .check_multi_interp_extensions = check_multi_interp_extensions, - .gil = gil, - }; + PyThreadState *substate; PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { /* Since no new thread state was created, there is no exception to @@ -1445,7 +1517,9 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } assert(substate != NULL); - r = PyRun_SimpleStringFlags(code, &cflags); + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + int r = PyRun_SimpleStringFlags(code, &cflags); Py_EndInterpreter(substate); PyThreadState_Swap(mainstate); @@ -1473,13 +1547,21 @@ unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } static PyObject * -new_interpreter(PyObject *self, PyObject *Py_UNUSED(ignored)) +new_interpreter(PyObject *self, PyObject *args) { + PyObject *configobj = NULL; + if (!PyArg_ParseTuple(args, "|O:new_interpreter", &configobj)) { + return NULL; + } + + PyInterpreterConfig config; + if (interp_config_from_object(configobj, &config) < 0) { + return NULL; + } + // Unlike _interpreters.create(), we do not automatically link // the interpreter to its refcount. PyThreadState *save_tstate = PyThreadState_Get(); - const PyInterpreterConfig config = \ - (PyInterpreterConfig)_PyInterpreterConfig_INIT; PyThreadState *tstate = NULL; PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); PyThreadState_Swap(save_tstate); @@ -1846,12 +1928,16 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"new_interp_config", _PyCFunction_CAST(new_interp_config), + METH_VARARGS | METH_KEYWORDS}, + {"get_interp_config", _PyCFunction_CAST(get_interp_config), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, {"normalize_interp_id", normalize_interp_id, METH_O}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, - {"new_interpreter", new_interpreter, METH_NOARGS}, + {"new_interpreter", new_interpreter, METH_VARARGS}, {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 82471e0f140ec3..9c82fcf021bb55 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -222,6 +222,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 97c52fdadf7c05..63b033a0350b20 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -229,6 +229,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 7a2a98df6511a1..657ffd1aa4c676 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -587,6 +587,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 89b56ec1267104..6e0cd1754f5cff 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1343,6 +1343,9 @@ Python + + Python + Source Files diff --git a/Python/config_common.h b/Python/config_common.h new file mode 100644 index 00000000000000..e749bd4bf0dc68 --- /dev/null +++ b/Python/config_common.h @@ -0,0 +1,36 @@ + +static inline int +_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) +{ + PyObject *item; + if (PyDict_GetItemStringRef(dict, name, &item) < 0) { + return -1; + } + if (item == NULL) { + // We do not set an exception. + return -1; + } + *p_item = item; + return 0; +} + + +static PyObject* +config_dict_get(PyObject *dict, const char *name) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + } + return NULL; + } + return item; +} + + +static void +config_dict_invalid_type(const char *name) +{ + PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); +} diff --git a/Python/initconfig.c b/Python/initconfig.c index 215d6a1d4e0dba..d91a8199b544dc 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -24,6 +24,9 @@ # endif #endif +#include "config_common.h" + + /* --- PyConfig spec ---------------------------------------------- */ typedef enum { @@ -1098,21 +1101,6 @@ _PyConfig_AsDict(const PyConfig *config) } -static PyObject* -config_dict_get(PyObject *dict, const char *name) -{ - PyObject *item; - if (PyDict_GetItemStringRef(dict, name, &item) < 0) { - return NULL; - } - if (item == NULL) { - PyErr_Format(PyExc_ValueError, "missing config key: %s", name); - return NULL; - } - return item; -} - - static void config_dict_invalid_value(const char *name) { @@ -1120,13 +1108,6 @@ config_dict_invalid_value(const char *name) } -static void -config_dict_invalid_type(const char *name) -{ - PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); -} - - static int config_dict_get_int(PyObject *dict, const char *name, int *result) { diff --git a/Python/interpconfig.c b/Python/interpconfig.c new file mode 100644 index 00000000000000..419f40ae62a89e --- /dev/null +++ b/Python/interpconfig.c @@ -0,0 +1,266 @@ +/* PyInterpreterConfig API */ + +#include "Python.h" +#include "pycore_pylifecycle.h" + +#include + +#include "config_common.h" + + +static const char * +gil_flag_to_str(int flag) +{ + switch (flag) { + case PyInterpreterConfig_DEFAULT_GIL: + return "default"; + case PyInterpreterConfig_SHARED_GIL: + return "shared"; + case PyInterpreterConfig_OWN_GIL: + return "own"; + default: + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return NULL; + } +} + +static int +gil_flag_from_str(const char *str, int *p_flag) +{ + int flag; + if (str == NULL) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "default") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "shared") == 0) { + flag = PyInterpreterConfig_SHARED_GIL; + } + else if (strcmp(str, "own") == 0) { + flag = PyInterpreterConfig_OWN_GIL; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported interpreter config .gil value '%s'", str); + return -1; + } + *p_flag = flag; + return 0; +} + +PyObject * +_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + +#define ADD(NAME, OBJ) \ + do { \ + int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ + Py_DECREF(OBJ); \ + if (res < 0) { \ + goto error; \ + } \ + } while (0) +#define ADD_BOOL(FIELD) \ + ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) +#define ADD_STR(FIELD, STR) \ + do { \ + if (STR == NULL) { \ + goto error; \ + } \ + PyObject *obj = PyUnicode_FromString(STR); \ + if (obj == NULL) { \ + goto error; \ + } \ + ADD(#FIELD, obj); \ + } while (0) + + ADD_BOOL(use_main_obmalloc); + ADD_BOOL(allow_fork); + ADD_BOOL(allow_exec); + ADD_BOOL(allow_threads); + ADD_BOOL(allow_daemon_threads); + ADD_BOOL(check_multi_interp_extensions); + + ADD_STR(gil, gil_flag_to_str(config->gil)); + +#undef ADD_STR +#undef ADD_BOOL +#undef ADD + + return dict; + +error: + Py_DECREF(dict); + return NULL; +} + +static int +_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + // For now we keep things strict, rather than using PyObject_IsTrue(). + int flag = item == Py_True; + if (!flag && item != Py_False) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + Py_DECREF(item); + *p_flag = flag; + return 0; +} + +static int +_config_dict_copy_str(PyObject *dict, const char *name, + char *buf, size_t bufsize) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + if (!PyUnicode_Check(item)) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); + buf[bufsize-1] = '\0'; + Py_DECREF(item); + return 0; +} + +static int +interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, + bool missing_allowed) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return -1; + } + if (PyDict_Update(dict, origdict) < 0) { + goto error; + } + +#define CHECK(NAME) \ + do { \ + if (PyErr_Occurred()) { \ + goto error; \ + } \ + else { \ + if (!missing_allowed) { \ + (void)config_dict_get(dict, NAME); \ + assert(PyErr_Occurred()); \ + goto error; \ + } \ + } \ + } while (0) +#define COPY_BOOL(FIELD) \ + do { \ + int flag; \ + if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ + CHECK(#FIELD); \ + } \ + else { \ + config->FIELD = flag; \ + (void)PyDict_PopString(dict, #FIELD, NULL); \ + } \ + } while (0) + + COPY_BOOL(use_main_obmalloc); + COPY_BOOL(allow_fork); + COPY_BOOL(allow_exec); + COPY_BOOL(allow_threads); + COPY_BOOL(allow_daemon_threads); + COPY_BOOL(check_multi_interp_extensions); + + // PyInterpreterConfig.gil + char buf[20]; + if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { + CHECK("gil"); + } + else { + int flag; + if (gil_flag_from_str(buf, &flag) < 0) { + goto error; + } + config->gil = flag; + (void)PyDict_PopString(dict, "gil", NULL); + } + +#undef COPY_BOOL +#undef CHECK + + Py_ssize_t unused = PyDict_GET_SIZE(dict); + if (unused == 1) { + PyErr_Format(PyExc_ValueError, + "config dict has 1 extra item (%R)", dict); + goto error; + } + else if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "config dict has %d extra items (%R)", unused, dict); + goto error; + } + return 0; + +error: + Py_DECREF(dict); + return -1; +} + +int +_PyInterpreterConfig_InitFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, false) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, true) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_InitFromState(PyInterpreterConfig *config, + PyInterpreterState *interp) +{ + // Populate the config by re-constructing the values from the interpreter. + *config = (PyInterpreterConfig){ +#define FLAG(flag) \ + (interp->feature_flags & Py_RTFLAGS_ ## flag) + .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), + .allow_fork = FLAG(FORK), + .allow_exec = FLAG(EXEC), + .allow_threads = FLAG(THREADS), + .allow_daemon_threads = FLAG(DAEMON_THREADS), + .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), +#undef FLAG + .gil = interp->ceval.own_gil + ? PyInterpreterConfig_OWN_GIL + : PyInterpreterConfig_SHARED_GIL, + }; + return 0; +} From 857d3151c9efa029268e8249e91d26eb1b31c2fd Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 17:16:50 -0600 Subject: [PATCH 055/143] gh-76785: Consolidate Some Interpreter-related Testing Helpers (gh-117485) This eliminates the duplication of functionally identical helpers in the _testinternalcapi and _xxsubinterpreters modules. --- Lib/test/support/interpreters/__init__.py | 8 +- Lib/test/test__xxsubinterpreters.py | 4 +- Lib/test/test_capi/test_misc.py | 83 +++---- Lib/test/test_import/__init__.py | 2 +- Lib/test/test_importlib/test_util.py | 4 +- Lib/test/test_interpreters/test_api.py | 211 +++++++++++++++- Lib/test/test_interpreters/test_queues.py | 6 +- Lib/test/test_interpreters/utils.py | 21 +- Modules/_testinternalcapi.c | 215 +--------------- Modules/_xxsubinterpretersmodule.c | 284 ++++++++++++++++++---- 10 files changed, 526 insertions(+), 312 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index d8e6654fc96efd..8316b5e4a93bb6 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -73,7 +73,7 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=True) + id = _interpreters.create(reqrefs=True) return Interpreter(id) @@ -109,13 +109,13 @@ def __new__(cls, id, /): assert hasattr(self, '_ownsref') except KeyError: # This may raise InterpreterNotFoundError: - _interpreters._incref(id) + _interpreters.incref(id) try: self = super().__new__(cls) self._id = id self._ownsref = True except BaseException: - _interpreters._deccref(id) + _interpreters.decref(id) raise _known[id] = self return self @@ -142,7 +142,7 @@ def _decref(self): return self._ownsref = False try: - _interpreters._decref(self.id) + _interpreters.decref(self.id) except InterpreterNotFoundError: pass diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 35d7355680e549..f674771c27cbb1 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -584,7 +584,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create(isolated=True) + subinterp = interpreters.create('isolated') script, file = _captured_script(f""" import threading def f(): @@ -604,7 +604,7 @@ def f(): self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create(isolated=False) + subinterp = interpreters.create('legacy') script, file = _captured_script(""" import threading def f(): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 34311afc93fc29..2f2bf03749f834 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2204,6 +2204,7 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +@requires_subinterpreters class InterpreterConfigTests(unittest.TestCase): supported = { @@ -2277,11 +2278,11 @@ def check(name, expected): expected = self.supported[expected] args = (name,) if name else () - config1 = _testinternalcapi.new_interp_config(*args) + config1 = _interpreters.new_config(*args) self.assert_ns_equal(config1, expected) self.assertIsNot(config1, expected) - config2 = _testinternalcapi.new_interp_config(*args) + config2 = _interpreters.new_config(*args) self.assert_ns_equal(config2, expected) self.assertIsNot(config2, expected) self.assertIsNot(config2, config1) @@ -2298,7 +2299,7 @@ def test_update_from_dict(self): with self.subTest(f'noop ({name})'): expected = vanilla overrides = vars(vanilla) - config = _testinternalcapi.new_interp_config(name, **overrides) + config = _interpreters.new_config(name, **overrides) self.assert_ns_equal(config, expected) with self.subTest(f'change all ({name})'): @@ -2308,7 +2309,7 @@ def test_update_from_dict(self): continue overrides['gil'] = gil expected = types.SimpleNamespace(**overrides) - config = _testinternalcapi.new_interp_config( + config = _interpreters.new_config( name, **overrides) self.assert_ns_equal(config, expected) @@ -2324,14 +2325,14 @@ def test_update_from_dict(self): expected = types.SimpleNamespace( **dict(vars(vanilla), **overrides), ) - config = _testinternalcapi.new_interp_config( + config = _interpreters.new_config( name, **overrides) self.assert_ns_equal(config, expected) with self.subTest('unsupported field'): for name in self.supported: with self.assertRaises(ValueError): - _testinternalcapi.new_interp_config(name, spam=True) + _interpreters.new_config(name, spam=True) # Bad values for bool fields. for field, value in vars(self.supported['empty']).items(): @@ -2341,19 +2342,18 @@ def test_update_from_dict(self): for value in [1, '', 'spam', 1.0, None, object()]: with self.subTest(f'unsupported value ({field}={value!r})'): with self.assertRaises(TypeError): - _testinternalcapi.new_interp_config(**{field: value}) + _interpreters.new_config(**{field: value}) # Bad values for .gil. for value in [True, 1, 1.0, None, object()]: with self.subTest(f'unsupported value(gil={value!r})'): with self.assertRaises(TypeError): - _testinternalcapi.new_interp_config(gil=value) + _interpreters.new_config(gil=value) for value in ['', 'spam']: with self.subTest(f'unsupported value (gil={value!r})'): with self.assertRaises(ValueError): - _testinternalcapi.new_interp_config(gil=value) + _interpreters.new_config(gil=value) - @requires_subinterpreters def test_interp_init(self): questionable = [ # strange @@ -2412,11 +2412,10 @@ def check(config): with self.subTest(f'valid: {config}'): check(config) - @requires_subinterpreters def test_get_config(self): @contextlib.contextmanager def new_interp(config): - interpid = _testinternalcapi.new_interpreter(config) + interpid = _interpreters.create(config, reqrefs=False) try: yield interpid finally: @@ -2426,32 +2425,32 @@ def new_interp(config): pass with self.subTest('main'): - expected = _testinternalcapi.new_interp_config('legacy') + expected = _interpreters.new_config('legacy') expected.gil = 'own' interpid = _interpreters.get_main() - config = _testinternalcapi.get_interp_config(interpid) + config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('isolated'): - expected = _testinternalcapi.new_interp_config('isolated') + expected = _interpreters.new_config('isolated') with new_interp('isolated') as interpid: - config = _testinternalcapi.get_interp_config(interpid) + config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('legacy'): - expected = _testinternalcapi.new_interp_config('legacy') + expected = _interpreters.new_config('legacy') with new_interp('legacy') as interpid: - config = _testinternalcapi.get_interp_config(interpid) + config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('custom'): - orig = _testinternalcapi.new_interp_config( + orig = _interpreters.new_config( 'empty', use_main_obmalloc=True, gil='shared', ) with new_interp(orig) as interpid: - config = _testinternalcapi.get_interp_config(interpid) + config = _interpreters.get_config(interpid) self.assert_ns_equal(config, orig) @@ -2529,14 +2528,19 @@ def test_lookup_destroyed(self): self.assertFalse( _testinternalcapi.interpreter_exists(interpid)) + def get_refcount_helpers(self): + return ( + _testinternalcapi.get_interpreter_refcount, + (lambda id: _interpreters.incref(id, implieslink=False)), + _interpreters.decref, + ) + def test_linked_lifecycle_does_not_exist(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + get_refcount, incref, decref = self.get_refcount_helpers() with self.subTest('never existed'): interpid = _testinternalcapi.unused_interpreter_id() @@ -2578,8 +2582,7 @@ def test_linked_lifecycle_initial(self): get_refcount = _testinternalcapi.get_interpreter_refcount # A new interpreter will start out not linked, with a refcount of 0. - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() linked = is_linked(interpid) refcount = get_refcount(interpid) @@ -2589,12 +2592,9 @@ def test_linked_lifecycle_initial(self): def test_linked_lifecycle_never_linked(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + get_refcount, incref, decref = self.get_refcount_helpers() - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() # Incref will not automatically link it. incref(interpid) @@ -2618,8 +2618,7 @@ def test_linked_lifecycle_link_unlink(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() # Linking at refcount 0 does not destroy the interpreter. link(interpid) @@ -2639,12 +2638,9 @@ def test_linked_lifecycle_link_incref_decref(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + get_refcount, incref, decref = self.get_refcount_helpers() - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() # Linking it will not change the refcount. link(interpid) @@ -2666,11 +2662,9 @@ def test_linked_lifecycle_link_incref_decref(self): def test_linked_lifecycle_incref_link(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref + get_refcount, incref, _ = self.get_refcount_helpers() - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() incref(interpid) self.assertEqual( @@ -2688,12 +2682,9 @@ def test_linked_lifecycle_link_incref_unlink_decref(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + get_refcount, incref, decref = self.get_refcount_helpers() - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() link(interpid) self.assertTrue( diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 81ec700d9755ce..3c387d973ce0f9 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2163,7 +2163,7 @@ def re_load(self, name, mod): # subinterpreters def add_subinterpreter(self): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index a6a76e589761e0..115cb7a56c98f7 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -656,7 +656,7 @@ def test_magic_number(self): class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): - interpid = _interpreters.create(isolated=True) + interpid = _interpreters.create('isolated') def ensure_destroyed(): try: _interpreters.destroy(interpid) @@ -669,7 +669,7 @@ def ensure_destroyed(): raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 3cde9bd0014d9a..2aa7f9bb61aa5b 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,13 +1,14 @@ import os import pickle -import threading from textwrap import dedent +import threading +import types import unittest from test import support from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +_interpreters = import_helper.import_module('_xxsubinterpreters') from test.support import interpreters from test.support.interpreters import InterpreterNotFoundError from .utils import _captured_script, _run_output, _running, TestBase @@ -932,6 +933,212 @@ class SubBytes(bytes): interpreters.is_shareable(obj)) +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_new_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + default = _interpreters.new_config('isolated') + with self.subTest('no arg'): + config = _interpreters.new_config() + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + with self.subTest('default'): + config1 = _interpreters.new_config('default') + self.assert_ns_equal(config1, default) + self.assertIsNot(config1, default) + + config2 = _interpreters.new_config('default') + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + for arg in ['', 'default']: + with self.subTest(f'default ({arg!r})'): + config = _interpreters.new_config(arg) + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=False, + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + for name, vanilla in supported.items(): + with self.subTest(f'supported ({name})'): + expected = vanilla + config1 = _interpreters.new_config(name) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(name) + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + with self.subTest(f'noop override ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'override all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('extra override'): + with self.assertRaises(ValueError): + _interpreters.new_config(spam=True) + + # Bad values for bool fields. + for field, value in vars(supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'bad override ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_get_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + interpid = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + interpid = _interpreters.create('isolated') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + interpid = _interpreters.create('legacy') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + def test_create(self): + isolated = _interpreters.new_config('isolated') + legacy = _interpreters.new_config('legacy') + default = isolated + + with self.subTest('no arg'): + interpid = _interpreters.create() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('arg: None'): + interpid = _interpreters.create(None) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('arg: \'empty\''): + with self.assertRaises(RuntimeError): + # The "empty" config isn't viable on its own. + _interpreters.create('empty') + + for arg, expected in { + '': default, + 'default': default, + 'isolated': isolated, + 'legacy': legacy, + }.items(): + with self.subTest(f'str arg: {arg!r}'): + interpid = _interpreters.create(arg) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config('empty') + orig.use_main_obmalloc = True + orig.gil = 'shared' + interpid = _interpreters.create(orig) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + with self.subTest('missing fields'): + orig = _interpreters.new_config() + del orig.gil + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('extra fields'): + orig = _interpreters.new_config() + orig.spam = True + with self.assertRaises(ValueError): + _interpreters.create(orig) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index d16d294b82d044..8ab9ebb354712a 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -28,9 +28,9 @@ def tearDown(self): class LowLevelTests(TestBase): - # The behaviors in the low-level module is important in as much - # as it is exercised by the high-level module. Therefore the - # most # important testing happens in the high-level tests. + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. # These low-level tests cover corner cases that are not # encountered by the high-level module, thus they # mostly shouldn't matter as much. diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 973d05d4f96dcb..5ade6762ea24ef 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -68,6 +68,9 @@ def run(): class TestBase(unittest.TestCase): + def tearDown(self): + clean_up_interpreters() + def pipe(self): def ensure_closed(fd): try: @@ -156,5 +159,19 @@ def assert_python_failure(self, *argv): self.assertNotEqual(exitcode, 0) return stdout, stderr - def tearDown(self): - clean_up_interpreters() + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 56761d1a896d2a..c5d65a373906f2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,7 +23,6 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() -#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() @@ -831,6 +830,7 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, } +// Maybe this could be replaced by get_interpreter_config()? static PyObject * get_interp_settings(PyObject *self, PyObject *args) { @@ -1357,129 +1357,6 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } -static int -init_named_interp_config(PyInterpreterConfig *config, const char *name) -{ - if (name == NULL) { - name = "isolated"; - } - - if (strcmp(name, "isolated") == 0) { - *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; - } - else if (strcmp(name, "legacy") == 0) { - *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - } - else if (strcmp(name, "empty") == 0) { - *config = (PyInterpreterConfig){0}; - } - else { - PyErr_Format(PyExc_ValueError, - "unsupported config name '%s'", name); - return -1; - } - return 0; -} - -static PyObject * -new_interp_config(PyObject *self, PyObject *args, PyObject *kwds) -{ - const char *name = NULL; - if (!PyArg_ParseTuple(args, "|s:new_config", &name)) { - return NULL; - } - PyObject *overrides = kwds; - - if (name == NULL) { - name = "isolated"; - } - - PyInterpreterConfig config; - if (init_named_interp_config(&config, name) < 0) { - return NULL; - } - - if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { - if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { - return NULL; - } - } - - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - -static PyObject * -get_interp_config(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", NULL}; - PyObject *idobj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:get_config", kwlist, &idobj)) - { - return NULL; - } - - PyInterpreterState *interp; - if (idobj == NULL) { - interp = PyInterpreterState_Get(); - } - else { - interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - } - - PyInterpreterConfig config; - if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { - return NULL; - } - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - -static int -interp_config_from_object(PyObject *configobj, PyInterpreterConfig *config) -{ - if (configobj == NULL || configobj == Py_None) { - if (init_named_interp_config(config, NULL) < 0) { - return -1; - } - } - else if (PyUnicode_Check(configobj)) { - if (init_named_interp_config(config, PyUnicode_AsUTF8(configobj)) < 0) { - return -1; - } - } - else { - PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); - if (dict == NULL) { - PyErr_Format(PyExc_TypeError, "bad config %R", configobj); - return -1; - } - int res = _PyInterpreterConfig_InitFromDict(config, dict); - Py_DECREF(dict); - if (res < 0) { - return -1; - } - } - return 0; -} - - /* To run some code in a sub-interpreter. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1495,7 +1372,14 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) } PyInterpreterConfig config; - if (interp_config_from_object(configobj, &config) < 0) { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return NULL; + } + int res = _PyInterpreterConfig_InitFromDict(&config, dict); + Py_DECREF(dict); + if (res < 0) { return NULL; } @@ -1546,58 +1430,6 @@ unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromLongLong(interpid); } -static PyObject * -new_interpreter(PyObject *self, PyObject *args) -{ - PyObject *configobj = NULL; - if (!PyArg_ParseTuple(args, "|O:new_interpreter", &configobj)) { - return NULL; - } - - PyInterpreterConfig config; - if (interp_config_from_object(configobj, &config) < 0) { - return NULL; - } - - // Unlike _interpreters.create(), we do not automatically link - // the interpreter to its refcount. - PyThreadState *save_tstate = PyThreadState_Get(); - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - _PyErr_SetFromPyStatus(status); - return NULL; - } - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - - if (_PyInterpreterState_IDInitref(interp) < 0) { - goto error; - } - - int64_t interpid = PyInterpreterState_GetID(interp); - if (interpid < 0) { - goto error; - } - PyObject *idobj = PyLong_FromLongLong(interpid); - if (idobj == NULL) { - goto error; - } - - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - - return idobj; - -error: - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; -} - static PyObject * interpreter_exists(PyObject *self, PyObject *idobj) { @@ -1660,28 +1492,6 @@ interpreter_refcount_linked(PyObject *self, PyObject *idobj) Py_RETURN_FALSE; } -static PyObject * -interpreter_incref(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - _PyInterpreterState_IDIncref(interp); - Py_RETURN_NONE; -} - -static PyObject * -interpreter_decref(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - _PyInterpreterState_IDDecref(interp); - Py_RETURN_NONE; -} - static void _xid_capsule_destructor(PyObject *capsule) @@ -1928,23 +1738,16 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, - {"new_interp_config", _PyCFunction_CAST(new_interp_config), - METH_VARARGS | METH_KEYWORDS}, - {"get_interp_config", _PyCFunction_CAST(get_interp_config), - METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, {"normalize_interp_id", normalize_interp_id, METH_O}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, - {"new_interpreter", new_interpreter, METH_VARARGS}, {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, - {"interpreter_incref", interpreter_incref, METH_O}, - {"interpreter_decref", interpreter_decref, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 5e5b3c10201867..9c2774e4f82def 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -12,8 +12,10 @@ #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -367,6 +369,115 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) /* interpreter-specific code ************************************************/ +static int +init_named_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL + || strcmp(name, "") == 0 + || strcmp(name, "default") == 0) + { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + +static int +config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + + +static PyInterpreterState * +new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate) +{ + PyThreadState *save_tstate = PyThreadState_Get(); + assert(save_tstate != NULL); + PyThreadState *tstate = NULL; + // XXX Possible GILState issues? + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + _PyErr_SetFromPyStatus(status); + return NULL; + } + assert(tstate != NULL); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + if (_PyInterpreterState_IDInitref(interp) < 0) { + goto error; + } + + if (p_idobj != NULL) { + // We create the object using the original interpreter. + PyObject *idobj = get_interpid_obj(interp); + if (idobj == NULL) { + goto error; + } + *p_idobj = idobj; + } + + if (p_tstate != NULL) { + *p_tstate = tstate; + } + else { + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + } + + return interp; + +error: + // XXX Possible GILState issues? + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; +} + + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -436,64 +547,98 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ static PyObject * -interp_create(PyObject *self, PyObject *args, PyObject *kwds) +interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) { + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", + &name)) + { + return NULL; + } + PyObject *overrides = kwds; - static char *kwlist[] = {"isolated", NULL}; - int isolated = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist, - &isolated)) { + PyInterpreterConfig config; + if (init_named_config(&config, name) < 0) { return NULL; } - // Create and initialize the new interpreter. - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - const PyInterpreterConfig config = isolated - ? (PyInterpreterConfig)_PyInterpreterConfig_INIT - : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } - // XXX Possible GILState issues? - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(new_config_doc, +"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ +\n\ +Return a representation of a new PyInterpreterConfig.\n\ +\n\ +The name determines the initial values of the config. Supported named\n\ +configs are: default, isolated, legacy, and empty.\n\ +\n\ +Any keyword arguments are set on the corresponding config fields,\n\ +overriding the initial values."); + + +static PyObject * +interp_create(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"config", "reqrefs", NULL}; + PyObject *configobj = NULL; + int reqrefs = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, + &configobj, &reqrefs)) { + return NULL; + } + + PyInterpreterConfig config; + if (config_from_object(configobj, &config) < 0) { + return NULL; + } + + PyObject *idobj = NULL; + PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL); + if (interp == NULL) { + // XXX Move the chained exception to interpreters.create()? PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); _PyErr_ChainExceptions1(exc); return NULL; } - assert(tstate != NULL); - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = get_interpid_obj(interp); - if (idobj == NULL) { - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; + if (reqrefs) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); } - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - - _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } + PyDoc_STRVAR(create_doc, -"create() -> ID\n\ +"create([config], *, reqrefs=False) -> ID\n\ \n\ Create a new interpreter and return a unique generated ID.\n\ \n\ -The caller is responsible for destroying the interpreter before exiting."); +The caller is responsible for destroying the interpreter before exiting,\n\ +typically by using _interpreters.destroy(). This can be managed \n\ +automatically by passing \"reqrefs=True\" and then using _incref() and\n\ +_decref()` appropriately.\n\ +\n\ +\"config\" must be a valid interpreter config or the name of a\n\ +predefined config (\"isolated\" or \"legacy\"). The default\n\ +is \"isolated\"."); static PyObject * @@ -1008,12 +1153,57 @@ Return whether or not the identified interpreter is running."); static PyObject * -interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:get_config", kwlist, &idobj)) + { + return NULL; + } + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(get_config_doc, +"get_config(id) -> types.SimpleNamespace\n\ +\n\ +Return a representation of the config used to initialize the interpreter."); + + +static PyObject * +interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "implieslink", NULL}; PyObject *id; + int implieslink = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O|$p:incref", kwlist, + &id, &implieslink)) + { return NULL; } @@ -1021,8 +1211,10 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - if (_PyInterpreterState_IDInitref(interp) < 0) { - return NULL; + + if (implieslink) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); } _PyInterpreterState_IDIncref(interp); @@ -1036,7 +1228,7 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) static char *kwlist[] = {"id", NULL}; PyObject *id; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O:decref", kwlist, &id)) { return NULL; } @@ -1051,6 +1243,8 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) static PyMethodDef module_functions[] = { + {"new_config", _PyCFunction_CAST(interp_new_config), + METH_VARARGS | METH_KEYWORDS, new_config_doc}, {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), @@ -1064,6 +1258,8 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, + {"get_config", _PyCFunction_CAST(interp_get_config), + METH_VARARGS | METH_KEYWORDS, get_config_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, {"call", _PyCFunction_CAST(interp_call), @@ -1078,9 +1274,9 @@ static PyMethodDef module_functions[] = { {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, - {"_incref", _PyCFunction_CAST(interp_incref), + {"incref", _PyCFunction_CAST(interp_incref), METH_VARARGS | METH_KEYWORDS, NULL}, - {"_decref", _PyCFunction_CAST(interp_decref), + {"decref", _PyCFunction_CAST(interp_decref), METH_VARARGS | METH_KEYWORDS, NULL}, {NULL, NULL} /* sentinel */ From 65524ab38875bb0b89fb499531bb772a4fb45b01 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 19:10:26 -0600 Subject: [PATCH 056/143] gh-76785: Fix a Refleak in _interpreters.new_config() (gh-117491) This is a follow-up to gh-117170 and gh-117485. --- Python/interpconfig.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/interpconfig.c b/Python/interpconfig.c index 419f40ae62a89e..54e5dca284c215 100644 --- a/Python/interpconfig.c +++ b/Python/interpconfig.c @@ -210,6 +210,8 @@ interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, "config dict has %d extra items (%R)", unused, dict); goto error; } + + Py_DECREF(dict); return 0; error: From 444156ede44204ef16c9d3cfcb03a637535fd5bf Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 3 Apr 2024 09:11:39 +0200 Subject: [PATCH 057/143] gh-117431: Adapt str.startswith and str.endswith to Argument Clinic (#117466) This change gives a significant speedup, as the METH_FASTCALL calling convention is now used. --- Lib/test/string_tests.py | 4 +- ...-04-02-17-37-35.gh-issue-117431.vDKAOn.rst | 2 + Objects/clinic/unicodeobject.c.h | 104 +++++++++++++++++- Objects/unicodeobject.c | 88 +++++++-------- 4 files changed, 149 insertions(+), 49 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index cecf309dca9194..5ade7013328d63 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -1513,9 +1513,9 @@ def test_find_etc_raise_correct_error_messages(self): x, None, None, None) self.assertRaisesRegex(TypeError, r'^count\(', s.count, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^startswith\(', s.startswith, + self.assertRaisesRegex(TypeError, r'^startswith\b', s.startswith, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^endswith\(', s.endswith, + self.assertRaisesRegex(TypeError, r'^endswith\b', s.endswith, x, None, None, None) # issue #15534 diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst new file mode 100644 index 00000000000000..96e14ea0c3b1bd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst @@ -0,0 +1,2 @@ +Improve the performance of :meth:`str.startswith` and :meth:`str.endswith` +by adapting them to the :c:macro:`METH_FASTCALL` calling convention. diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 3e5167d9242fe4..c956bb1936776a 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -1369,6 +1369,108 @@ unicode_zfill(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(unicode_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the string starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A string or a tuple of strings to try.\n" +" start\n" +" Optional start position. Default: start of the string.\n" +" end\n" +" Optional stop position. Default: end of the string."); + +#define UNICODE_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(unicode_startswith), METH_FASTCALL, unicode_startswith__doc__}, + +static PyObject * +unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_startswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = unicode_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_endswith__doc__, +"endswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the string ends with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A string or a tuple of strings to try.\n" +" start\n" +" Optional start position. Default: start of the string.\n" +" end\n" +" Optional stop position. Default: end of the string."); + +#define UNICODE_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(unicode_endswith), METH_FASTCALL, unicode_endswith__doc__}, + +static PyObject * +unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_endswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = unicode_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode___format____doc__, "__format__($self, format_spec, /)\n" "--\n" @@ -1507,4 +1609,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=1aab29bab5201c78 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e495e878d8283217 input=a9049054013a1b77]*/ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e412af5f797e7a..ac59419b3c7a50 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13021,30 +13021,30 @@ unicode_zfill_impl(PyObject *self, Py_ssize_t width) return u; } -PyDoc_STRVAR(startswith__doc__, - "S.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if S starts with the specified prefix, False otherwise.\n\ -With optional start, test S beginning at that position.\n\ -With optional end, stop comparing S at that position.\n\ -prefix can also be a tuple of strings to try."); +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +str.startswith as unicode_startswith + + prefix as subobj: object + A string or a tuple of strings to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the string. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the string. + / + +Return True if the string starts with the specified prefix, False otherwise. +[clinic start generated code]*/ static PyObject * -unicode_startswith(PyObject *self, - PyObject *args) +unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=4bd7cfd0803051d4 input=5f918b5f5f89d856]*/ { - PyObject *subobj; - PyObject *substring; - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - int result; - - if (!asciilib_parse_args_finds("startswith", args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); + PyObject *substring = PyTuple_GET_ITEM(subobj, i); if (!PyUnicode_Check(substring)) { PyErr_Format(PyExc_TypeError, "tuple for startswith must only contain str, " @@ -13052,9 +13052,10 @@ unicode_startswith(PyObject *self, Py_TYPE(substring)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, -1); - if (result == -1) + int result = tailmatch(self, substring, start, end, -1); + if (result < 0) { return NULL; + } if (result) { Py_RETURN_TRUE; } @@ -13068,37 +13069,30 @@ unicode_startswith(PyObject *self, "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, -1); - if (result == -1) + int result = tailmatch(self, subobj, start, end, -1); + if (result < 0) { return NULL; + } return PyBool_FromLong(result); } -PyDoc_STRVAR(endswith__doc__, - "S.endswith(suffix[, start[, end]]) -> bool\n\ -\n\ -Return True if S ends with the specified suffix, False otherwise.\n\ -With optional start, test S beginning at that position.\n\ -With optional end, stop comparing S at that position.\n\ -suffix can also be a tuple of strings to try."); +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +str.endswith as unicode_endswith = str.startswith + +Return True if the string ends with the specified prefix, False otherwise. +[clinic start generated code]*/ static PyObject * -unicode_endswith(PyObject *self, - PyObject *args) +unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=cce6f8ceb0102ca9 input=82cd5ce9e7623646]*/ { - PyObject *subobj; - PyObject *substring; - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - int result; - - if (!asciilib_parse_args_finds("endswith", args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); + PyObject *substring = PyTuple_GET_ITEM(subobj, i); if (!PyUnicode_Check(substring)) { PyErr_Format(PyExc_TypeError, "tuple for endswith must only contain str, " @@ -13106,9 +13100,10 @@ unicode_endswith(PyObject *self, Py_TYPE(substring)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, +1); - if (result == -1) + int result = tailmatch(self, substring, start, end, +1); + if (result < 0) { return NULL; + } if (result) { Py_RETURN_TRUE; } @@ -13121,9 +13116,10 @@ unicode_endswith(PyObject *self, "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, +1); - if (result == -1) + int result = tailmatch(self, subobj, start, end, +1); + if (result < 0) { return NULL; + } return PyBool_FromLong(result); } @@ -13576,8 +13572,8 @@ static PyMethodDef unicode_methods[] = { UNICODE_SWAPCASE_METHODDEF UNICODE_TRANSLATE_METHODDEF UNICODE_UPPER_METHODDEF - {"startswith", (PyCFunction) unicode_startswith, METH_VARARGS, startswith__doc__}, - {"endswith", (PyCFunction) unicode_endswith, METH_VARARGS, endswith__doc__}, + UNICODE_STARTSWITH_METHODDEF + UNICODE_ENDSWITH_METHODDEF UNICODE_REMOVEPREFIX_METHODDEF UNICODE_REMOVESUFFIX_METHODDEF UNICODE_ISASCII_METHODDEF From 8987a5c809343ae0dd2b8e607bf2c32a87773127 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Wed, 3 Apr 2024 10:43:52 +0200 Subject: [PATCH 058/143] gh-91565: Update issue tracker URL in error message. (#117450) * Update issue tracker URL in commit message. * Also update issue tracker URL in comment. --- Modules/selectmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index f16173aafa7d3c..6ea141ab1f9189 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -817,10 +817,10 @@ static int devpoll_flush(devpollObject *self) ** clear what to do if a partial write occurred. For now, raise ** an exception and see if we actually found this problem in ** the wild. - ** See http://bugs.python.org/issue6397. + ** See https://github.com/python/cpython/issues/50646. */ PyErr_Format(PyExc_OSError, "failed to write all pollfds. " - "Please, report at http://bugs.python.org/. " + "Please, report at https://github.com/python/cpython/issues/. " "Data to report: Size tried: %d, actual size written: %d.", size, n); return -1; From 8ef98924d304b5c9430e23f8170e2c32ec3a9920 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 3 Apr 2024 12:18:05 +0200 Subject: [PATCH 059/143] gh-117281: Change weakref repr() to fully qualified name (#117285) Use the fully qualified type name in repr() of weakref.ref and weakref.proxy types. Fix a crash in proxy_repr() when the reference is dead. Add also test_ref_repr() and test_proxy_repr(). --- Lib/test/test_weakref.py | 41 ++++++++++++++++++++++++++++++++++++++++ Objects/weakrefobject.c | 24 +++++++++++++++-------- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 4cdd66d3769e0c..d6470fb6919cfd 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -116,6 +116,33 @@ def test_basic_ref(self): del o repr(wr) + @support.cpython_only + def test_ref_repr(self): + obj = C() + ref = weakref.ref(obj) + self.assertRegex(repr(ref), + rf"") + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'') + + # test type with __name__ + class WithName: + @property + def __name__(self): + return "custom_name" + + obj2 = WithName() + ref2 = weakref.ref(obj2) + self.assertRegex(repr(ref2), + rf"") + def test_repr_failure_gh99184(self): class MyConfig(dict): def __getattr__(self, x): @@ -195,6 +222,20 @@ def check(proxy): self.assertRaises(ReferenceError, bool, ref3) self.assertEqual(self.cbcalled, 2) + @support.cpython_only + def test_proxy_repr(self): + obj = C() + ref = weakref.proxy(obj, self.callback) + self.assertRegex(repr(ref), + rf"") + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'') + def check_basic_ref(self, factory): o = factory() ref = weakref.ref(o) diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index b7b29064151609..d8dd6aea3aff02 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -177,13 +177,13 @@ weakref_repr(PyObject *self) PyObject *repr; if (name == NULL || !PyUnicode_Check(name)) { repr = PyUnicode_FromFormat( - "", - self, Py_TYPE(obj)->tp_name, obj); + "", + self, obj, obj); } else { repr = PyUnicode_FromFormat( - "", - self, Py_TYPE(obj)->tp_name, obj, name); + "", + self, obj, obj, name); } Py_DECREF(obj); Py_XDECREF(name); @@ -471,10 +471,18 @@ static PyObject * proxy_repr(PyObject *proxy) { PyObject *obj = _PyWeakref_GET_REF(proxy); - PyObject *repr = PyUnicode_FromFormat( - "", - proxy, Py_TYPE(obj)->tp_name, obj); - Py_DECREF(obj); + PyObject *repr; + if (obj != NULL) { + repr = PyUnicode_FromFormat( + "", + proxy, obj, obj); + Py_DECREF(obj); + } + else { + repr = PyUnicode_FromFormat( + "", + proxy); + } return repr; } From 1dc1521042d5e750b4a129ac8dd439edafed6783 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 3 Apr 2024 12:33:20 +0200 Subject: [PATCH 060/143] gh-117431: Fix str.endswith docstring (#117499) The first parameter is named 'suffix', not 'prefix'. Regression introduced by commit 444156ed --- Objects/clinic/unicodeobject.c.h | 8 ++++---- Objects/unicodeobject.c | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index c956bb1936776a..3f6dd8b93a7155 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -1421,12 +1421,12 @@ unicode_startswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(unicode_endswith__doc__, -"endswith($self, prefix[, start[, end]], /)\n" +"endswith($self, suffix[, start[, end]], /)\n" "--\n" "\n" -"Return True if the string ends with the specified prefix, False otherwise.\n" +"Return True if the string ends with the specified suffix, False otherwise.\n" "\n" -" prefix\n" +" suffix\n" " A string or a tuple of strings to try.\n" " start\n" " Optional start position. Default: start of the string.\n" @@ -1609,4 +1609,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=e495e878d8283217 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1734aa1fcc9b076a input=a9049054013a1b77]*/ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index ac59419b3c7a50..eb83312e9c3a69 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13078,16 +13078,24 @@ unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, /*[clinic input] -@text_signature "($self, prefix[, start[, end]], /)" -str.endswith as unicode_endswith = str.startswith +@text_signature "($self, suffix[, start[, end]], /)" +str.endswith as unicode_endswith + + suffix as subobj: object + A string or a tuple of strings to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the string. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the string. + / -Return True if the string ends with the specified prefix, False otherwise. +Return True if the string ends with the specified suffix, False otherwise. [clinic start generated code]*/ static PyObject * unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=cce6f8ceb0102ca9 input=82cd5ce9e7623646]*/ +/*[clinic end generated code: output=cce6f8ceb0102ca9 input=00fbdc774a7d4d71]*/ { if (PyTuple_Check(subobj)) { Py_ssize_t i; From 595bb496b0504429cf01a76fd1ada718d9dd25ca Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 3 Apr 2024 13:11:14 +0200 Subject: [PATCH 061/143] gh-117431: Adapt bytes and bytearray .startswith() and .endswith() to Argument Clinic (#117495) This change gives a significant speedup, as the METH_FASTCALL calling convention is now used. --- Include/internal/pycore_bytes_methods.h | 8 +- ...-04-03-09-49-15.gh-issue-117431.WAqRgc.rst | 6 + Objects/bytearrayobject.c | 50 +++++++-- Objects/bytes_methods.c | 51 +++------ Objects/bytesobject.c | 50 +++++++-- Objects/clinic/bytearrayobject.c.h | 104 +++++++++++++++++- Objects/clinic/bytesobject.c.h | 104 +++++++++++++++++- 7 files changed, 318 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-03-09-49-15.gh-issue-117431.WAqRgc.rst diff --git a/Include/internal/pycore_bytes_methods.h b/Include/internal/pycore_bytes_methods.h index 11e8ab20e91367..b9c0a4e2b2f77d 100644 --- a/Include/internal/pycore_bytes_methods.h +++ b/Include/internal/pycore_bytes_methods.h @@ -32,8 +32,12 @@ extern PyObject *_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args extern PyObject *_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args); extern PyObject *_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args); extern int _Py_bytes_contains(const char *str, Py_ssize_t len, PyObject *arg); -extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, + PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); +extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, + PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-03-09-49-15.gh-issue-117431.WAqRgc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-03-09-49-15.gh-issue-117431.WAqRgc.rst new file mode 100644 index 00000000000000..17374d0d5c575b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-03-09-49-15.gh-issue-117431.WAqRgc.rst @@ -0,0 +1,6 @@ +Improve the performance of the following :class:`bytes` and +:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` +calling convention: + +* :meth:`!endswith` +* :meth:`!startswith` diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 5e3b3affbc76c5..8639496727536a 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1186,16 +1186,52 @@ bytearray_contains(PyObject *self, PyObject *arg) return _Py_bytes_contains(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), arg); } +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +bytearray.startswith + + prefix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytearray. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytearray. + / + +Return True if the bytearray starts with the specified prefix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytearray_startswith(PyByteArrayObject *self, PyObject *args) +bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=a3d9b6d44d3662a6 input=76385e0b376b45c1]*/ { - return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subobj, start, end); } +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +bytearray.endswith + + suffix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytearray. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytearray. + / + +Return True if the bytearray ends with the specified suffix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytearray_endswith(PyByteArrayObject *self, PyObject *args) +bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=e75ea8c227954caa input=9b8baa879aa3d74b]*/ { - return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subobj, start, end); } /*[clinic input] @@ -2203,8 +2239,7 @@ bytearray_methods[] = { {"count", (PyCFunction)bytearray_count, METH_VARARGS, _Py_count__doc__}, BYTEARRAY_DECODE_METHODDEF - {"endswith", (PyCFunction)bytearray_endswith, METH_VARARGS, - _Py_endswith__doc__}, + BYTEARRAY_ENDSWITH_METHODDEF STRINGLIB_EXPANDTABS_METHODDEF BYTEARRAY_EXTEND_METHODDEF {"find", (PyCFunction)bytearray_find, METH_VARARGS, @@ -2249,8 +2284,7 @@ bytearray_methods[] = { BYTEARRAY_RSTRIP_METHODDEF BYTEARRAY_SPLIT_METHODDEF BYTEARRAY_SPLITLINES_METHODDEF - {"startswith", (PyCFunction)bytearray_startswith, METH_VARARGS , - _Py_startswith__doc__}, + BYTEARRAY_STARTSWITH_METHODDEF BYTEARRAY_STRIP_METHODDEF {"swapcase", stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index c1bc6383df30ce..21b6668171bf61 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -771,66 +771,47 @@ tailmatch(const char *str, Py_ssize_t len, PyObject *substr, static PyObject * _Py_bytes_tailmatch(const char *str, Py_ssize_t len, - const char *function_name, PyObject *args, + const char *function_name, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end, int direction) { - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj = NULL; - int result; - - if (!stringlib_parse_args_finds(function_name, args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = tailmatch(str, len, PyTuple_GET_ITEM(subobj, i), - start, end, direction); - if (result == -1) + PyObject *item = PyTuple_GET_ITEM(subobj, i); + int result = tailmatch(str, len, item, start, end, direction); + if (result < 0) { return NULL; + } else if (result) { Py_RETURN_TRUE; } } Py_RETURN_FALSE; } - result = tailmatch(str, len, subobj, start, end, direction); + int result = tailmatch(str, len, subobj, start, end, direction); if (result == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) + if (PyErr_ExceptionMatches(PyExc_TypeError)) { PyErr_Format(PyExc_TypeError, "%s first arg must be bytes or a tuple of bytes, " "not %s", function_name, Py_TYPE(subobj)->tp_name); + } return NULL; } - else - return PyBool_FromLong(result); + return PyBool_FromLong(result); } -PyDoc_STRVAR_shared(_Py_startswith__doc__, -"B.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if B starts with the specified prefix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -prefix can also be a tuple of bytes to try."); - PyObject * -_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) { - return _Py_bytes_tailmatch(str, len, "startswith", args, -1); + return _Py_bytes_tailmatch(str, len, "startswith", subobj, start, end, -1); } -PyDoc_STRVAR_shared(_Py_endswith__doc__, -"B.endswith(suffix[, start[, end]]) -> bool\n\ -\n\ -Return True if B ends with the specified suffix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -suffix can also be a tuple of bytes to try."); - PyObject * -_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) { - return _Py_bytes_tailmatch(str, len, "endswith", args, +1); + return _Py_bytes_tailmatch(str, len, "endswith", subobj, start, end, +1); } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 256e01f54f0782..d7b0c6b7b01aa9 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2285,16 +2285,52 @@ bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) return PyBytes_FromStringAndSize(self_start, self_len); } +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +bytes.startswith + + prefix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return True if the bytes starts with the specified prefix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytes_startswith(PyBytesObject *self, PyObject *args) +bytes_startswith_impl(PyBytesObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=b1e8da1cbd528e8c input=8a4165df8adfa6c9]*/ { - return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subobj, start, end); } +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +bytes.endswith + + suffix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return True if the bytes ends with the specified suffix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytes_endswith(PyBytesObject *self, PyObject *args) +bytes_endswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=038b633111f3629d input=b5c3407a2a5c9aac]*/ { - return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subobj, start, end); } @@ -2491,8 +2527,7 @@ bytes_methods[] = { {"count", (PyCFunction)bytes_count, METH_VARARGS, _Py_count__doc__}, BYTES_DECODE_METHODDEF - {"endswith", (PyCFunction)bytes_endswith, METH_VARARGS, - _Py_endswith__doc__}, + BYTES_ENDSWITH_METHODDEF STRINGLIB_EXPANDTABS_METHODDEF {"find", (PyCFunction)bytes_find, METH_VARARGS, _Py_find__doc__}, @@ -2532,8 +2567,7 @@ bytes_methods[] = { BYTES_RSTRIP_METHODDEF BYTES_SPLIT_METHODDEF BYTES_SPLITLINES_METHODDEF - {"startswith", (PyCFunction)bytes_startswith, METH_VARARGS, - _Py_startswith__doc__}, + BYTES_STARTSWITH_METHODDEF BYTES_STRIP_METHODDEF {"swapcase", stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index d95245067e2608..dabc2b16c94fce 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -137,6 +137,108 @@ bytearray_copy(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) return bytearray_copy_impl(self); } +PyDoc_STRVAR(bytearray_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytearray starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytearray.\n" +" end\n" +" Optional stop position. Default: end of the bytearray."); + +#define BYTEARRAY_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(bytearray_startswith), METH_FASTCALL, bytearray_startswith__doc__}, + +static PyObject * +bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_startswith(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytearray ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytearray.\n" +" end\n" +" Optional stop position. Default: end of the bytearray."); + +#define BYTEARRAY_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(bytearray_endswith), METH_FASTCALL, bytearray_endswith__doc__}, + +static PyObject * +bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_endswith(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_removeprefix__doc__, "removeprefix($self, prefix, /)\n" "--\n" @@ -1261,4 +1363,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=0797a5e03cda2a16 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0147908e97ebe882 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 1e45be3e7aefb3..05e182778aece1 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -652,6 +652,108 @@ bytes_removesuffix(PyBytesObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(bytes_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytes starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(bytes_startswith), METH_FASTCALL, bytes_startswith__doc__}, + +static PyObject * +bytes_startswith_impl(PyBytesObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytes_startswith(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytes ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(bytes_endswith), METH_FASTCALL, bytes_endswith__doc__}, + +static PyObject * +bytes_endswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_endswith(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytes_decode__doc__, "decode($self, /, encoding=\'utf-8\', errors=\'strict\')\n" "--\n" @@ -1029,4 +1131,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8a49dbbd78914a6f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f2b10ccd2e3155c3 input=a9049054013a1b77]*/ From 33ee5cb3e92ea8798e7f1a2f3a13b92b39cee6d6 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 3 Apr 2024 05:19:49 -0700 Subject: [PATCH 062/143] GH-70647: Deprecate strptime day of month parsing without a year present to avoid leap-year bugs (GH-117107) --- Doc/library/datetime.rst | 37 +++++++++++++++++++ Lib/_strptime.py | 21 ++++++++++- Lib/test/datetimetester.py | 13 +++++++ Lib/test/test_time.py | 8 ++++ Lib/test/test_unittest/test_assertions.py | 10 +++++ Lib/unittest/case.py | 22 +++++++++++ ...4-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst | 7 ++++ 7 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 1905c9e1ca755d..047427d3269027 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1079,6 +1079,24 @@ Other constructors, all class methods: time tuple. See also :ref:`strftime-strptime-behavior` and :meth:`datetime.fromisoformat`. + .. versionchanged:: 3.13 + + If *format* specifies a day of month without a year a + :exc:`DeprecationWarning` is now emitted. This is to avoid a quadrennial + leap year bug in code seeking to parse only a month and day as the + default year used in absence of one in the format is not a leap year. + Such *format* values may raise an error as of Python 3.15. The + workaround is to always include a year in your *format*. If parsing + *date_string* values that do not have a year, explicitly add a year that + is a leap year before parsing: + + .. doctest:: + + >>> from datetime import datetime + >>> date_string = "02/29" + >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when.strftime("%B %d") # doctest: +SKIP + 'February 29' Class attributes: @@ -2657,6 +2675,25 @@ Notes: for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. +(10) + When parsing a month and day using :meth:`~.datetime.strptime`, always + include a year in the format. If the value you need to parse lacks a year, + append an explicit dummy leap year. Otherwise your code will raise an + exception when it encounters leap day because the default year used by the + parser is not a leap year. Users run into this bug every four years... + + .. doctest:: + + >>> month_day = "02/29" + >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. + datetime.datetime(1984, 2, 29, 0, 0) + + .. deprecated-removed:: 3.13 3.15 + :meth:`~.datetime.strptime` calls using a format string containing + a day of month without a year now emit a + :exc:`DeprecationWarning`. In 3.15 or later we may change this into + an error or change the default year to a leap year. See :gh:`70647`. + .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 798cf9f9d3fffe..e42af75af74bf5 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -10,6 +10,7 @@ strptime -- Calculates the time struct represented by the passed-in string """ +import os import time import locale import calendar @@ -250,12 +251,30 @@ def pattern(self, format): format = regex_chars.sub(r"\\\1", format) whitespace_replacement = re_compile(r'\s+') format = whitespace_replacement.sub(r'\\s+', format) + year_in_format = False + day_of_month_in_format = False while '%' in format: directive_index = format.index('%')+1 + format_char = format[directive_index] processed_format = "%s%s%s" % (processed_format, format[:directive_index-1], - self[format[directive_index]]) + self[format_char]) format = format[directive_index+1:] + match format_char: + case 'Y' | 'y' | 'G': + year_in_format = True + case 'd': + day_of_month_in_format = True + if day_of_month_in_format and not year_in_format: + import warnings + warnings.warn("""\ +Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647.""", + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) return "%s%s" % (processed_format, format) def compile(self, format): diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 31fc383e29707a..c77263998c99f5 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2793,6 +2793,19 @@ def test_strptime_single_digit(self): newdate = strptime(string, format) self.assertEqual(newdate, target, msg=reason) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + self.theclass.strptime('02-29', '%m-%d') + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + self.theclass.strptime('03-14.159265', '%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('20-03-14.159265', '%y-%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('02-29,2024', '%m-%d,%Y') + def test_more_timetuple(self): # This tests fields beyond those tested by the TestDate.test_timetuple. t = self.theclass(2004, 12, 31, 6, 22, 33) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index fb234b7bc5962a..293799ff68ea05 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -277,6 +277,8 @@ def test_strptime(self): 'j', 'm', 'M', 'p', 'S', 'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'): format = '%' + directive + if directive == 'd': + format += ',%Y' # Avoid GH-70647. strf_output = time.strftime(format, tt) try: time.strptime(strf_output, format) @@ -299,6 +301,12 @@ def test_strptime_exception_context(self): time.strptime('19', '%Y %') self.assertIs(e.exception.__suppress_context__, True) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + time.strptime('02-07 18:28', '%m-%d %H:%M') + def test_asctime(self): time.asctime(time.gmtime(self.t)) diff --git a/Lib/test/test_unittest/test_assertions.py b/Lib/test/test_unittest/test_assertions.py index 5c1a28ecda5b49..1dec947ea76d23 100644 --- a/Lib/test/test_unittest/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -386,6 +386,16 @@ def testAssertWarns(self): '^UserWarning not triggered$', '^UserWarning not triggered : oops$']) + def test_assertNotWarns(self): + def warn_future(): + warnings.warn('xyz', FutureWarning, stacklevel=2) + self.assertMessagesCM('_assertNotWarns', (FutureWarning,), + warn_future, + ['^FutureWarning triggered$', + '^oops$', + '^FutureWarning triggered$', + '^FutureWarning triggered : oops$']) + def testAssertWarnsRegex(self): # test error not raised self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'unused regex'), diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 001b640dc43ad6..36daa61fa31adb 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -332,6 +332,23 @@ def __exit__(self, exc_type, exc_value, tb): self._raiseFailure("{} not triggered".format(exc_name)) +class _AssertNotWarnsContext(_AssertWarnsContext): + + def __exit__(self, exc_type, exc_value, tb): + self.warnings_manager.__exit__(exc_type, exc_value, tb) + if exc_type is not None: + # let unexpected exceptions pass through + return + try: + exc_name = self.expected.__name__ + except AttributeError: + exc_name = str(self.expected) + for m in self.warnings: + w = m.message + if isinstance(w, self.expected): + self._raiseFailure(f"{exc_name} triggered") + + class _OrderedChainMap(collections.ChainMap): def __iter__(self): seen = set() @@ -811,6 +828,11 @@ def assertWarns(self, expected_warning, *args, **kwargs): context = _AssertWarnsContext(expected_warning, self) return context.handle('assertWarns', args, kwargs) + def _assertNotWarns(self, expected_warning, *args, **kwargs): + """The opposite of assertWarns. Private due to low demand.""" + context = _AssertNotWarnsContext(expected_warning, self) + return context.handle('_assertNotWarns', args, kwargs) + def assertLogs(self, logger=None, level=None): """Fail unless a log message of level *level* or higher is emitted on *logger_name* or its children. If omitted, *level* defaults to diff --git a/Misc/NEWS.d/next/Library/2024-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst b/Misc/NEWS.d/next/Library/2024-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst new file mode 100644 index 00000000000000..a9094df06037cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst @@ -0,0 +1,7 @@ +Start the deprecation period for the current behavior of +:func:`datetime.datetime.strptime` and :func:`time.strptime` which always +fails to parse a date string with a :exc:`ValueError` involving a day of +month such as ``strptime("02-29", "%m-%d")`` when a year is **not** +specified and the date happen to be February 29th. This should help avoid +users finding new bugs every four years due to a natural mistaken assumption +about the API when parsing partial date values. From a214f55b274df9782e78e99516a372e0b800162a Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 3 Apr 2024 08:29:02 -0400 Subject: [PATCH 063/143] gh-117483: Accept "Broken pipe" as valid error message in `test_wrong_cert_tls13` (GH-117484) On macOS, the closed connection can lead to a "Broken pipe" error instead of a "Connection reset by peer" error. --- Lib/test/test_ssl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 794944afd66dd0..0e50d09c8f28d6 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -3169,7 +3169,9 @@ def test_wrong_cert_tls13(self): s.connect((HOST, server.port)) with self.assertRaisesRegex( OSError, - 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA|closed by the remote host|Connection reset by peer' + 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA|' + 'closed by the remote host|Connection reset by peer|' + 'Broken pipe' ): # TLS 1.3 perform client cert exchange after handshake s.write(b'data') From 2ec6bb4111d2c03c1cac02b27c74beee7e5a2a05 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 3 Apr 2024 15:10:09 +0200 Subject: [PATCH 064/143] gh-117381: Improve error messages for ntpath.commonpath() (GH-117382) --- Lib/ntpath.py | 9 +- Lib/test/test_ntpath.py | 97 +++++++++---------- ...-03-29-21-43-19.gh-issue-117381.fT0JFM.rst | 1 + 3 files changed, 54 insertions(+), 53 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-29-21-43-19.gh-issue-117381.fT0JFM.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 0650f14f89f10b..f9f6c78566e8ed 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -857,9 +857,6 @@ def commonpath(paths): drivesplits = [splitroot(p.replace(altsep, sep).lower()) for p in paths] split_paths = [p.split(sep) for d, r, p in drivesplits] - if len({r for d, r, p in drivesplits}) != 1: - raise ValueError("Can't mix absolute and relative paths") - # Check that all drive letters or UNC paths match. The check is made only # now otherwise type errors for mixing strings and bytes would not be # caught. @@ -867,6 +864,12 @@ def commonpath(paths): raise ValueError("Paths don't have the same drive") drive, root, path = splitroot(paths[0].replace(altsep, sep)) + if len({r for d, r, p in drivesplits}) != 1: + if drive: + raise ValueError("Can't mix absolute and relative paths") + else: + raise ValueError("Can't mix rooted and not-rooted paths") + common = path.split(sep) common = [c for c in common if c and c != curdir] diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index c816f99e7e9f1b..31156130fcc747 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -866,46 +866,47 @@ def test_commonpath(self): def check(paths, expected): tester(('ntpath.commonpath(%r)' % paths).replace('\\\\', '\\'), expected) - def check_error(exc, paths): - self.assertRaises(exc, ntpath.commonpath, paths) - self.assertRaises(exc, ntpath.commonpath, - [os.fsencode(p) for p in paths]) + def check_error(paths, expected): + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths[::-1]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths[::-1]]) self.assertRaises(TypeError, ntpath.commonpath, None) self.assertRaises(ValueError, ntpath.commonpath, []) self.assertRaises(ValueError, ntpath.commonpath, iter([])) - check_error(ValueError, ['C:\\Program Files', 'Program Files']) - check_error(ValueError, ['C:\\Program Files', 'C:Program Files']) - check_error(ValueError, ['\\Program Files', 'Program Files']) - check_error(ValueError, ['Program Files', 'C:\\Program Files']) - - check(['C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files'], - 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files\\'], - 'C:\\Program Files') - check(['C:\\\\Program Files', 'C:\\Program Files\\\\'], - 'C:\\Program Files') - check(['C:\\.\\Program Files', 'C:\\Program Files\\.'], - 'C:\\Program Files') - check(['C:\\', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Projects'], 'C:\\') - check(['C:\\Program Files\\', 'C:\\Projects'], 'C:\\') - - check(['C:\\Program Files\\Foo', 'C:/Program Files/Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'c:/program files/bar'], - 'C:\\Program Files') - check(['c:/program files/bar', 'C:\\Program Files\\Foo'], - 'c:\\program files') - - check_error(ValueError, ['C:\\Program Files', 'D:\\Program Files']) + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'C:Foo'], "Can't mix absolute and relative paths") + check_error(['C:\\Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'Foo'], "Paths don't have the same drive") + check_error(['C:Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'Foo'], "Paths don't have the same drive") + check_error(['\\Foo', 'Foo'], "Can't mix rooted and not-rooted paths") + + check(['C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo\\'], 'C:\\Foo') + check(['C:\\\\Foo', 'C:\\Foo\\\\'], 'C:\\Foo') + check(['C:\\.\\Foo', 'C:\\Foo\\.'], 'C:\\Foo') + check(['C:\\', 'C:\\baz'], 'C:\\') + check(['C:\\Bar', 'C:\\baz'], 'C:\\') + check(['C:\\Foo', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Bar', 'C:\\Baz'], 'C:\\') + check(['C:\\Bar\\', 'C:\\Baz'], 'C:\\') + + check(['C:\\Foo\\Bar', 'C:/Foo/Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'c:/foo/baz'], 'C:\\Foo') + check(['c:/foo/bar', 'C:\\Foo\\Baz'], 'c:\\foo') + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'D:\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'D:Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'D:Foo'], "Paths don't have the same drive") check(['spam'], 'spam') check(['spam', 'spam'], 'spam') @@ -919,20 +920,16 @@ def check_error(exc, paths): check([''], '') check(['', 'spam\\alot'], '') - check_error(ValueError, ['', '\\spam\\alot']) - - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['Program Files', b'C:\\Program Files\\Foo']) + + # gh-117381: Logical error messages + check_error(['', '\\spam\\alot'], "Can't mix rooted and not-rooted paths") + + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['Foo', b'C:\\Foo\\Baz']) @unittest.skipIf(is_emscripten, "Emscripten cannot fstat unnamed files.") def test_sameopenfile(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-29-21-43-19.gh-issue-117381.fT0JFM.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-29-21-43-19.gh-issue-117381.fT0JFM.rst new file mode 100644 index 00000000000000..88b6c32e971e72 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-29-21-43-19.gh-issue-117381.fT0JFM.rst @@ -0,0 +1 @@ +Fix error message for :func:`ntpath.commonpath`. From ea94b3b149eeadf33c2f7c46f16dcda0adc7cf4e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 3 Apr 2024 15:11:36 +0200 Subject: [PATCH 065/143] gh-116303: Skip test module dependent tests if test modules are unavailable (#117341) --- Lib/test/support/__init__.py | 4 + Lib/test/test_audit.py | 1 + Lib/test/test_call.py | 81 +++++++++++---------- Lib/test/test_capi/__init__.py | 7 ++ Lib/test/test_code.py | 7 +- Lib/test/test_coroutines.py | 5 ++ Lib/test/test_ctypes/test_as_parameter.py | 3 +- Lib/test/test_ctypes/test_bitfields.py | 3 +- Lib/test/test_ctypes/test_callbacks.py | 3 +- Lib/test/test_ctypes/test_cfuncs.py | 3 +- Lib/test/test_ctypes/test_checkretval.py | 3 +- Lib/test/test_ctypes/test_funcptr.py | 3 +- Lib/test/test_ctypes/test_functions.py | 3 +- Lib/test/test_ctypes/test_libc.py | 3 +- Lib/test/test_ctypes/test_loading.py | 2 +- Lib/test/test_ctypes/test_parameters.py | 3 +- Lib/test/test_ctypes/test_pickling.py | 3 +- Lib/test/test_ctypes/test_pointers.py | 3 +- Lib/test/test_ctypes/test_prototypes.py | 3 +- Lib/test/test_ctypes/test_refcounts.py | 3 +- Lib/test/test_ctypes/test_returnfuncptrs.py | 3 +- Lib/test/test_ctypes/test_slicing.py | 3 +- Lib/test/test_ctypes/test_stringptr.py | 3 +- Lib/test/test_ctypes/test_structures.py | 3 +- Lib/test/test_ctypes/test_unicode.py | 3 +- Lib/test/test_ctypes/test_values.py | 10 ++- Lib/test/test_ctypes/test_win32.py | 4 +- Lib/test/test_embed.py | 12 ++- Lib/test/test_exceptions.py | 18 +++-- Lib/test/test_external_inspection.py | 2 +- Lib/test/test_faulthandler.py | 1 + Lib/test/test_fcntl.py | 6 +- Lib/test/test_fileio.py | 3 +- Lib/test/test_finalization.py | 4 +- Lib/test/test_format.py | 4 +- Lib/test/test_gc.py | 10 +-- Lib/test/test_genericclass.py | 5 +- Lib/test/test_import/__init__.py | 12 ++- Lib/test/test_importlib/util.py | 2 + Lib/test/test_inspect/test_inspect.py | 17 +++-- Lib/test/test_monitoring.py | 5 +- Lib/test/test_multibytecodec.py | 3 +- Lib/test/test_opcache.py | 3 +- Lib/test/test_os.py | 3 + Lib/test/test_perfmaps.py | 6 +- Lib/test/test_poll.py | 5 +- Lib/test/test_pydoc/test_pydoc.py | 12 +-- Lib/test/test_regrtest.py | 8 ++ Lib/test/test_repl.py | 2 + Lib/test/test_socket.py | 24 ++++-- Lib/test/test_sqlite3/test_dbapi.py | 18 +++-- Lib/test/test_stable_abi_ctypes.py | 5 +- Lib/test/test_sys.py | 2 +- Lib/test/test_threading.py | 4 +- Lib/test/test_tools/test_makefile.py | 1 + Lib/test/test_weakref.py | 3 +- Tools/build/stable_abi.py | 5 +- 57 files changed, 256 insertions(+), 124 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 9640d5d831b874..d7bc416ab04086 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1168,6 +1168,10 @@ def requires_limited_api(test): return unittest.skip('needs _testcapi and _testlimitedcapi modules')(test) return test + +TEST_MODULES_ENABLED = sysconfig.get_config_var('TEST_MODULES') == 'yes' + + def requires_specialization(test): return unittest.skipUnless( _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index c24c8213924196..e163c7ad25cc7b 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -89,6 +89,7 @@ def test_excepthook(self): ) def test_unraisablehook(self): + import_helper.import_module("_testcapi") returncode, events, stderr = self.run_python("test_unraisablehook") if returncode: self.fail(stderr) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index eaf919584b4c64..d3f4d6c29c5536 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,6 +1,6 @@ import unittest from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, - set_recursion_limit, skip_on_s390x) + set_recursion_limit, skip_on_s390x, import_helper) try: import _testcapi except ImportError: @@ -244,6 +244,7 @@ def test_module_not_callable_suggestion(self): self.assertRaisesRegex(TypeError, msg, mod) +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestCallingConventions(unittest.TestCase): """Test calling using various C calling conventions (METH_*) from Python @@ -441,6 +442,7 @@ def static_method(): NULL_OR_EMPTY = object() + class FastCallTests(unittest.TestCase): """Test calling using various callables from C """ @@ -484,42 +486,43 @@ class FastCallTests(unittest.TestCase): ] # Add all the calling conventions and variants of C callables - _instance = _testcapi.MethInstance() - for obj, expected_self in ( - (_testcapi, _testcapi), # module-level function - (_instance, _instance), # bound method - (_testcapi.MethClass, _testcapi.MethClass), # class method on class - (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. - (_testcapi.MethStatic, None), # static method - ): - CALLS_POSARGS.extend([ - (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), - (obj.meth_varargs_keywords, - (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), - (obj.meth_fastcall, (), (expected_self, ())), - (obj.meth_fastcall_keywords, - (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall_keywords, - (), (expected_self, (), NULL_OR_EMPTY)), - (obj.meth_noargs, (), expected_self), - (obj.meth_o, (123, ), (expected_self, 123)), - ]) - - CALLS_KWARGS.extend([ - (obj.meth_varargs_keywords, - (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), - (obj.meth_varargs_keywords, - (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), - (obj.meth_varargs_keywords, - (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall_keywords, - (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), - (obj.meth_fastcall_keywords, - (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), - (obj.meth_fastcall_keywords, - (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), - ]) + if _testcapi: + _instance = _testcapi.MethInstance() + for obj, expected_self in ( + (_testcapi, _testcapi), # module-level function + (_instance, _instance), # bound method + (_testcapi.MethClass, _testcapi.MethClass), # class method on class + (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. + (_testcapi.MethStatic, None), # static method + ): + CALLS_POSARGS.extend([ + (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), + (obj.meth_varargs_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), + (obj.meth_fastcall, (), (expected_self, ())), + (obj.meth_fastcall_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (), (expected_self, (), NULL_OR_EMPTY)), + (obj.meth_noargs, (), expected_self), + (obj.meth_o, (123, ), (expected_self, 123)), + ]) + + CALLS_KWARGS.extend([ + (obj.meth_varargs_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_varargs_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_varargs_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + ]) def check_result(self, result, expected): if isinstance(expected, tuple) and expected[-1] is NULL_OR_EMPTY: @@ -527,6 +530,7 @@ def check_result(self, result, expected): expected = (*expected[:-1], result[-1]) self.assertEqual(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall_dict(self): # Test PyObject_VectorcallDict() @@ -546,6 +550,7 @@ def test_vectorcall_dict(self): result = _testcapi.pyobject_fastcalldict(func, args, kwargs) self.check_result(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall(self): # Test PyObject_Vectorcall() @@ -610,6 +615,7 @@ def testfunction_kw(self, *, kw): ADAPTIVE_WARMUP_DELAY = 2 +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestPEP590(unittest.TestCase): def test_method_descriptor_flag(self): @@ -1022,6 +1028,7 @@ class TestRecursion(unittest.TestCase): @skip_on_s390x @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_capi/__init__.py b/Lib/test/test_capi/__init__.py index 4b16ecc31156a5..5a8ba6845399e0 100644 --- a/Lib/test/test_capi/__init__.py +++ b/Lib/test/test_capi/__init__.py @@ -1,5 +1,12 @@ import os +import unittest from test.support import load_package_tests +from test.support import TEST_MODULES_ENABLED + + +if not TEST_MODULES_ENABLED: + raise unittest.SkipTest("requires test modules") + def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 46bebfc7af675b..ecd1e82a6dbef9 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -143,9 +143,8 @@ check_impl_detail, requires_debug_ranges, gc_collect) from test.support.script_helper import assert_python_ok -from test.support import threading_helper -from test.support.bytecode_helper import (BytecodeTestCase, - instructions_with_positions) +from test.support import threading_helper, import_helper +from test.support.bytecode_helper import instructions_with_positions from opcode import opmap, opname COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -176,7 +175,7 @@ class CodeTest(unittest.TestCase): @cpython_only def test_newempty(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") co = _testcapi.code_newempty("filename", "funcname", 15) self.assertEqual(co.co_filename, "filename") self.assertEqual(co.co_name, "funcname") diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index d848bfbd46c83b..f705f4f5bfbd88 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -11,6 +11,10 @@ from test.support import import_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok +try: + import _testcapi +except ImportError: + _testcapi = None class AsyncYieldFrom: @@ -2445,6 +2449,7 @@ def test_unawaited_warning_during_shutdown(self): @support.cpython_only +@unittest.skipIf(_testcapi is None, "requires _testcapi") class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index ca75e748256083..cc62b1a22a3b06 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import unittest from ctypes import (Structure, CDLL, CFUNCTYPE, @@ -6,6 +5,8 @@ c_short, c_int, c_long, c_longlong, c_byte, c_wchar, c_float, c_double, ArgumentError) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index d43c56ad371fbd..0332544b5827e6 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -1,4 +1,3 @@ -import _ctypes_test import os import unittest from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, @@ -7,6 +6,8 @@ c_uint32, c_uint64, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class BITS(Structure): diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index 8038169afe4304..8f483dfe1db801 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import functools import gc @@ -14,6 +13,8 @@ c_float, c_double, c_longdouble, py_object) from ctypes.util import find_library from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class Callbacks(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_cfuncs.py b/Lib/test/test_ctypes/test_cfuncs.py index 6ff0878a35da2f..48330c4b0a763b 100644 --- a/Lib/test/test_ctypes/test_cfuncs.py +++ b/Lib/test/test_ctypes/test_cfuncs.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import unittest from ctypes import (CDLL, @@ -6,6 +5,8 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class CFunctions(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_checkretval.py b/Lib/test/test_ctypes/test_checkretval.py index 5dc9e25aa38226..9d6bfdb845e6c7 100644 --- a/Lib/test/test_ctypes/test_checkretval.py +++ b/Lib/test/test_ctypes/test_checkretval.py @@ -1,7 +1,8 @@ -import _ctypes_test import ctypes import unittest from ctypes import CDLL, c_int +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class CHECKED(c_int): diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index 03cfddea6ea61a..8362fb16d94dcd 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -1,8 +1,9 @@ -import _ctypes_test import ctypes import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, sizeof, _CFuncPtr, c_void_p, c_char_p, c_char, c_int, c_uint, c_long) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) diff --git a/Lib/test/test_ctypes/test_functions.py b/Lib/test/test_ctypes/test_functions.py index 04e8582ff1e427..63e393f7b7cb6a 100644 --- a/Lib/test/test_ctypes/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import sys import unittest @@ -7,6 +6,8 @@ c_char, c_wchar, c_byte, c_char_p, c_wchar_p, c_short, c_int, c_long, c_longlong, c_void_p, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from _ctypes import _Pointer, _SimpleCData diff --git a/Lib/test/test_ctypes/test_libc.py b/Lib/test/test_ctypes/test_libc.py index 09c76db0bd0b17..7716100b08f78e 100644 --- a/Lib/test/test_ctypes/test_libc.py +++ b/Lib/test/test_ctypes/test_libc.py @@ -1,8 +1,9 @@ -import _ctypes_test import math import unittest from ctypes import (CDLL, CFUNCTYPE, POINTER, create_string_buffer, sizeof, c_void_p, c_char, c_int, c_double, c_size_t) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") lib = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index b218e9e7720c79..b25e81b65cf103 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -1,5 +1,4 @@ import _ctypes -import _ctypes_test import ctypes import os import shutil @@ -10,6 +9,7 @@ from ctypes import CDLL, cdll, addressof, c_void_p, c_char_p from ctypes.util import find_library from test.support import import_helper, os_helper +_ctypes_test = import_helper.import_module("_ctypes_test") libc_name = None diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index d1eeee6b0306fe..effb8db418f790 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,4 +1,3 @@ -import _ctypes_test import unittest import test.support from ctypes import (CDLL, PyDLL, ArgumentError, @@ -14,6 +13,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class SimpleTypesTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_pickling.py b/Lib/test/test_ctypes/test_pickling.py index 0ca42a68f0675f..9d433fc69de391 100644 --- a/Lib/test/test_ctypes/test_pickling.py +++ b/Lib/test/test_ctypes/test_pickling.py @@ -1,9 +1,10 @@ -import _ctypes_test import pickle import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, pointer, c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, c_int, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 3a5f3660dbbe23..fc558e10ba40c5 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -1,4 +1,3 @@ -import _ctypes_test import array import ctypes import sys @@ -10,6 +9,8 @@ c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCPointerType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) diff --git a/Lib/test/test_ctypes/test_prototypes.py b/Lib/test/test_ctypes/test_prototypes.py index 81eb4562c740fd..63ae799ea86ab2 100644 --- a/Lib/test/test_ctypes/test_prototypes.py +++ b/Lib/test/test_ctypes/test_prototypes.py @@ -18,12 +18,13 @@ # # In this case, there would have to be an additional reference to the argument... -import _ctypes_test import unittest from ctypes import (CDLL, CFUNCTYPE, POINTER, ArgumentError, pointer, byref, sizeof, addressof, create_string_buffer, c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, c_short, c_int, c_long, c_longlong, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") testdll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_refcounts.py b/Lib/test/test_ctypes/test_refcounts.py index a90588ca9bb1b6..e6427d4a295b15 100644 --- a/Lib/test/test_ctypes/test_refcounts.py +++ b/Lib/test/test_ctypes/test_refcounts.py @@ -1,9 +1,10 @@ -import _ctypes_test import ctypes import gc import sys import unittest from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") MyCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int) diff --git a/Lib/test/test_ctypes/test_returnfuncptrs.py b/Lib/test/test_ctypes/test_returnfuncptrs.py index 4010e511e75ade..337801b226ab06 100644 --- a/Lib/test/test_ctypes/test_returnfuncptrs.py +++ b/Lib/test/test_ctypes/test_returnfuncptrs.py @@ -1,6 +1,7 @@ -import _ctypes_test import unittest from ctypes import CDLL, CFUNCTYPE, ArgumentError, c_char_p, c_void_p, c_char +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class ReturnFuncPtrTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_slicing.py b/Lib/test/test_ctypes/test_slicing.py index a592d911cbe6ca..66f9e530104cac 100644 --- a/Lib/test/test_ctypes/test_slicing.py +++ b/Lib/test/test_ctypes/test_slicing.py @@ -1,7 +1,8 @@ -import _ctypes_test import unittest from ctypes import (CDLL, POINTER, sizeof, c_byte, c_short, c_int, c_long, c_char, c_wchar, c_char_p) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class SlicesTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_stringptr.py b/Lib/test/test_ctypes/test_stringptr.py index 67c61c6c3e17e6..bb6045b250ffce 100644 --- a/Lib/test/test_ctypes/test_stringptr.py +++ b/Lib/test/test_ctypes/test_stringptr.py @@ -1,9 +1,10 @@ -import _ctypes_test import sys import unittest from test import support from ctypes import (CDLL, Structure, POINTER, create_string_buffer, c_char, c_char_p) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") lib = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 8d83ce4f281b16..7650c80273f812 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -1,4 +1,3 @@ -import _ctypes_test from platform import architecture as _architecture import struct import sys @@ -12,6 +11,8 @@ from struct import calcsize from collections import namedtuple from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCStructType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) diff --git a/Lib/test/test_ctypes/test_unicode.py b/Lib/test/test_ctypes/test_unicode.py index 2ddc7c56544e35..d9e17371d13572 100644 --- a/Lib/test/test_ctypes/test_unicode.py +++ b/Lib/test/test_ctypes/test_unicode.py @@ -1,6 +1,7 @@ -import _ctypes_test import ctypes import unittest +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class UnicodeTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_values.py b/Lib/test/test_ctypes/test_values.py index d0b4803dff8529..1b757e020d5ce2 100644 --- a/Lib/test/test_ctypes/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -2,7 +2,6 @@ A testcase which accesses *values* in a dll. """ -import _ctypes_test import _imp import importlib.util import sys @@ -15,10 +14,14 @@ class ValuesTestCase(unittest.TestCase): + def setUp(self): + _ctypes_test = import_helper.import_module("_ctypes_test") + self.ctdll = CDLL(_ctypes_test.__file__) + def test_an_integer(self): # This test checks and changes an integer stored inside the # _ctypes_test dll/shared lib. - ctdll = CDLL(_ctypes_test.__file__) + ctdll = self.ctdll an_integer = c_int.in_dll(ctdll, "an_integer") x = an_integer.value self.assertEqual(x, ctdll.get_an_integer()) @@ -30,8 +33,7 @@ def test_an_integer(self): self.assertEqual(x, ctdll.get_an_integer()) def test_undefined(self): - ctdll = CDLL(_ctypes_test.__file__) - self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") + self.assertRaises(ValueError, c_int.in_dll, self.ctdll, "Undefined_Symbol") class PythonValuesTestCase(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_win32.py b/Lib/test/test_ctypes/test_win32.py index 4aaecd8d38f98f..31919118670613 100644 --- a/Lib/test/test_ctypes/test_win32.py +++ b/Lib/test/test_ctypes/test_win32.py @@ -1,6 +1,5 @@ # Windows specific tests -import _ctypes_test import ctypes import errno import sys @@ -9,6 +8,7 @@ _pointer_type_cache, c_void_p, c_char, c_int, c_long) from test import support +from test.support import import_helper from ._support import Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE @@ -36,6 +36,7 @@ def test_noargs(self): @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class ReturnStructSizesTestCase(unittest.TestCase): def test_sizes(self): + _ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) for i in range(1, 11): fields = [ (f"f{f}", c_char) for f in range(1, i + 1)] @@ -116,6 +117,7 @@ class RECT(Structure): ("right", c_long), ("bottom", c_long)] + _ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) pt = POINT(15, 25) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index ab1d579ed12755..ec928f935655f9 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -19,6 +19,13 @@ if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") + +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + + MACOS = (sys.platform == 'darwin') PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 @@ -352,6 +359,7 @@ def test_simple_initialization_api(self): self.assertEqual(out, 'Finalized\n' * INIT_LOOPS) @support.requires_specialization + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): # https://github.com/python/cpython/issues/92031 @@ -396,6 +404,8 @@ def test_ucnhash_capi_reset(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '9\n' * INIT_LOOPS) + +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): maxDiff = 4096 UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape') @@ -1588,7 +1598,6 @@ def test_global_pathconfig(self): # The global path configuration (_Py_path_config) must be a copy # of the path configuration of PyInterpreter.config (PyConfig). ctypes = import_helper.import_module('ctypes') - _testinternalcapi = import_helper.import_module('_testinternalcapi') def get_func(name): func = getattr(ctypes.pythonapi, name) @@ -1784,6 +1793,7 @@ def test_unicode_id_init(self): # See bpo-44133 @unittest.skipIf(os.name == 'nt', 'Py_FrozenMain is not exported on Windows') + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_frozenmain(self): env = dict(os.environ) env['PYTHONUNBUFFERED'] = '1' diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c5eff8ad8ccca1..6ad6acc61563e5 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -19,12 +19,13 @@ from test import support try: + import _testcapi from _testcapi import INT_MAX except ImportError: + _testcapi = None INT_MAX = 2**31 - 1 - class NaiveException(Exception): def __init__(self, x): self.x = x @@ -345,8 +346,8 @@ def __init__(self_): class InvalidException: pass + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi1(): - import _testcapi try: _testcapi.raise_exception(BadException, 1) except TypeError as err: @@ -356,8 +357,8 @@ def test_capi1(): else: self.fail("Expected exception") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi2(): - import _testcapi try: _testcapi.raise_exception(BadException, 0) except RuntimeError as err: @@ -370,8 +371,8 @@ def test_capi2(): else: self.fail("Expected exception") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi3(): - import _testcapi self.assertRaises(SystemError, _testcapi.raise_exception, InvalidException, 1) @@ -1381,6 +1382,7 @@ def foo(): @cpython_only def test_recursion_normalizing_exception(self): + import_module("_testinternalcapi") # Issue #22898. # Test that a RecursionError is raised when tstate->recursion_depth is # equal to recursion_limit in PyErr_NormalizeException() and check @@ -1435,6 +1437,7 @@ def gen(): self.assertIn(b'Done.', out) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_recursion_normalizing_infinite_exception(self): # Issue #30697. Test that a RecursionError is raised when # maximum recursion depth has been exceeded when creating @@ -1503,6 +1506,7 @@ def recurse_in_body_and_except(): # Python built with Py_TRACE_REFS fail with a fatal error in # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no # memory left and the size of the Python frames stack is greater than @@ -1525,6 +1529,7 @@ def recurse(cnt): self.assertIn(b'MemoryError', err) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_MemoryError(self): # PyErr_NoMemory always raises the same exception instance. # Check that the traceback is not doubled. @@ -1544,8 +1549,8 @@ def raiseMemError(): self.assertEqual(tb1, tb2) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_exception_with_doc(self): - import _testcapi doc2 = "This is a test docstring." doc4 = "This is another test docstring." @@ -1584,6 +1589,7 @@ class C(object): self.assertEqual(error5.__doc__, "") @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_cleanup(self): # Issue #5437: preallocated MemoryError instances should not keep # traceback objects alive. @@ -1674,6 +1680,7 @@ def test_unhandled(self): # Python built with Py_TRACE_REFS fail with a fatal error in # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_in_PyErr_PrintEx(self): code = """if 1: import _testcapi @@ -1792,6 +1799,7 @@ class TestException(MemoryError): gc_collect() + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_in_subinterp(self): # gh-109894: subinterpreters shouldn't count on last resort memory error # when MemoryError is raised through PyErr_NoMemory() call, diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 86c07de507e39c..d896fec73d1971 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -14,7 +14,7 @@ from _testexternalinspection import PROCESS_VM_READV_SUPPORTED from _testexternalinspection import get_stack_trace except ImportError: - unittest.skip("Test only runs when _testexternalinspection is available") + raise unittest.SkipTest("Test only runs when _testexternalinspection is available") def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index d0473500a17735..200f34d18ca60a 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -266,6 +266,7 @@ def test_sigill(self): 5, 'Illegal instruction') + @unittest.skipIf(_testcapi is None, 'need _testcapi') def check_fatal_error_func(self, release_gil): # Test that Py_FatalError() dumps a traceback with support.SuppressCrashReport(): diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 5fae0de7423c87..84c0e5bc4800a1 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -117,7 +117,9 @@ def test_fcntl_bad_file(self): @cpython_only def test_fcntl_bad_file_overflow(self): - from _testcapi import INT_MAX, INT_MIN + _testcapi = import_module("_testcapi") + INT_MAX = _testcapi.INT_MAX + INT_MIN = _testcapi.INT_MIN # Issue 15989 with self.assertRaises(OverflowError): fcntl.fcntl(INT_MAX + 1, fcntl.F_SETFL, os.O_NONBLOCK) @@ -189,7 +191,7 @@ def test_lockf_share(self): @cpython_only def test_flock_overflow(self): - import _testcapi + _testcapi = import_module("_testcapi") self.assertRaises(OverflowError, fcntl.flock, _testcapi.INT_MAX+1, fcntl.LOCK_SH) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 06d5a8abf32083..0611d1749f41c1 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -17,6 +17,7 @@ TESTFN, TESTFN_ASCII, TESTFN_UNICODE, make_bad_fd, ) from test.support.warnings_helper import check_warnings +from test.support.import_helper import import_module from collections import UserList import _io # C implementation of io @@ -597,7 +598,7 @@ class COtherFileTests(OtherFileTests, unittest.TestCase): @cpython_only def testInvalidFd_overflow(self): # Issue 15989 - import _testcapi + _testcapi = import_module("_testcapi") self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MAX + 1) self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MIN - 1) diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py index 1d134430909d84..42871f8a09b16b 100644 --- a/Lib/test/test_finalization.py +++ b/Lib/test/test_finalization.py @@ -13,7 +13,7 @@ def with_tp_del(cls): class C(object): def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.with_tp_del') + raise unittest.SkipTest('requires _testcapi.with_tp_del') return C try: @@ -22,7 +22,7 @@ def __new__(cls, *args, **kwargs): def without_gc(cls): class C: def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.without_gc') + raise unittest.SkipTest('requires _testcapi.without_gc') return C from test import support diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 6fa49dbc0b730c..8cef621bd716ac 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -4,6 +4,7 @@ import re import test.support as support import unittest +from test.support.import_helper import import_module maxsize = support.MAX_Py_ssize_t @@ -478,7 +479,8 @@ def test_precision(self): @support.cpython_only def test_precision_c_limits(self): - from _testcapi import INT_MAX + _testcapi = import_module("_testcapi") + INT_MAX = _testcapi.INT_MAX f = 1.2 with self.assertRaises(ValueError) as cm: diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index fa8e50fccb2c7b..3a01013b771082 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -16,17 +16,16 @@ import weakref try: + import _testcapi from _testcapi import with_tp_del + from _testcapi import ContainerNoGC except ImportError: + _testcapi = None def with_tp_del(cls): class C(object): def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.with_tp_del') + raise unittest.SkipTest('requires _testcapi.with_tp_del') return C - -try: - from _testcapi import ContainerNoGC -except ImportError: ContainerNoGC = None ### Support code @@ -681,6 +680,7 @@ def do_work(): @cpython_only @requires_subprocess() + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_garbage_at_shutdown(self): import subprocess code = """if 1: diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index aece757fc1933e..e530b463966819 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -1,5 +1,6 @@ import unittest from test import support +from test.support.import_helper import import_module class TestMROEntry(unittest.TestCase): @@ -277,7 +278,9 @@ def __class_getitem__(cls, item): class CAPITest(unittest.TestCase): def test_c_class(self): - from _testcapi import Generic, GenericAlias + _testcapi = import_module("_testcapi") + Generic = _testcapi.Generic + GenericAlias = _testcapi.GenericAlias self.assertIsInstance(Generic.__class_getitem__(int), GenericAlias) IntGeneric = Generic[int] diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 3c387d973ce0f9..6678548a0ffaca 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -33,7 +33,7 @@ is_wasi, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, ready_to_import, - DirsOnSysPath, CleanImport) + DirsOnSysPath, CleanImport, import_module) from test.support.os_helper import ( TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE) from test.support import script_helper @@ -363,7 +363,7 @@ def test_from_import_missing_attr_has_name_and_path(self): @cpython_only def test_from_import_missing_attr_has_name_and_so_path(self): - import _testcapi + _testcapi = import_module("_testcapi") with self.assertRaises(ImportError) as cm: from _testcapi import i_dont_exist self.assertEqual(cm.exception.name, '_testcapi') @@ -1870,6 +1870,7 @@ def check_incompatible_fresh(self, name, *, isolated=False): f'ImportError: module {name} does not support loading in subinterpreters', ) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_builtin_compat(self): # For now we avoid using sys or builtins # since they still don't implement multi-phase init. @@ -1881,6 +1882,7 @@ def test_builtin_compat(self): self.check_compatible_here(module, strict=True) @cpython_only + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_frozen_compat(self): module = '_frozen_importlib' require_frozen(module, skip=True) @@ -1951,6 +1953,7 @@ def test_multi_init_extension_per_interpreter_gil_compat(self): self.check_compatible_here(modname, filename, strict=False, isolated=False) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_python_compat(self): module = 'threading' require_pure_python(module) @@ -1996,6 +1999,7 @@ def check_incompatible(setting, override): with self.subTest('config: check disabled; override: disabled'): check_compatible(False, -1) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_isolated_config(self): module = 'threading' require_pure_python(module) @@ -2741,7 +2745,7 @@ class CAPITests(unittest.TestCase): def test_pyimport_addmodule(self): # gh-105922: Test PyImport_AddModuleRef(), PyImport_AddModule() # and PyImport_AddModuleObject() - import _testcapi + _testcapi = import_module("_testcapi") for name in ( 'sys', # frozen module 'test', # package @@ -2751,7 +2755,7 @@ def test_pyimport_addmodule(self): def test_pyimport_addmodule_create(self): # gh-105922: Test PyImport_AddModuleRef(), create a new module - import _testcapi + _testcapi = import_module("_testcapi") name = 'dontexist' self.assertNotIn(name, sys.modules) self.addCleanup(unload, name) diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 89272484009c56..edbe78545a2536 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -15,6 +15,8 @@ import tempfile import types +_testsinglephase = import_helper.import_module("_testsinglephase") + BUILTINS = types.SimpleNamespace() BUILTINS.good_name = None diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index dc46c0bc8ed353..6494842c217662 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -32,7 +32,7 @@ except ImportError: ThreadPoolExecutor = None -from test.support import cpython_only +from test.support import cpython_only, import_helper from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support.import_helper import DirsOnSysPath, ready_to_import from test.support.os_helper import TESTFN, temp_cwd @@ -668,7 +668,10 @@ def test_cleandoc(self): @cpython_only def test_c_cleandoc(self): - import _testinternalcapi + try: + import _testinternalcapi + except ImportError: + return unittest.skip("requires _testinternalcapi") func = _testinternalcapi.compiler_cleandoc for i, (input, expected) in enumerate(self.cleandoc_testdata): with self.subTest(i=i): @@ -1220,7 +1223,7 @@ def test_getfullargspec_builtin_methods(self): @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_func(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.docstring_with_signature_with_defaults spec = inspect.getfullargspec(builtin) self.assertEqual(spec.defaults[0], 'avocado') @@ -1229,7 +1232,7 @@ def test_getfullargspec_builtin_func(self): @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_func_no_signature(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.docstring_no_signature with self.assertRaises(TypeError): inspect.getfullargspec(builtin) @@ -2890,7 +2893,7 @@ def test_staticmethod(*args): # NOQA @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_builtins(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") def test_unbound_method(o): """Use this to test unbound methods (things that should have a self)""" @@ -2971,7 +2974,7 @@ class ThisWorksNow: @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_decorated_builtins(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") func = _testcapi.docstring_with_signature_with_defaults def decorator(func): @@ -2992,7 +2995,7 @@ def wrapper_like(*args, **kwargs) -> int: pass @cpython_only def test_signature_on_builtins_no_signature(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") with self.assertRaisesRegex(ValueError, 'no signature found for builtin'): inspect.signature(_testcapi.docstring_no_signature) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 58441ef8b82fd0..11c61bc2e0688d 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -11,6 +11,7 @@ import asyncio from test import support from test.support import requires_specialization, script_helper +from test.support.import_helper import import_module PAIR = (0,1) @@ -1829,15 +1830,15 @@ def f(a=1, b=2): class TestOptimizer(MonitoringTestBase, unittest.TestCase): def setUp(self): - import _testinternalcapi + _testinternalcapi = import_module("_testinternalcapi") self.old_opt = _testinternalcapi.get_optimizer() opt = _testinternalcapi.new_counter_optimizer() _testinternalcapi.set_optimizer(opt) super(TestOptimizer, self).setUp() def tearDown(self): - import _testinternalcapi super(TestOptimizer, self).tearDown() + import _testinternalcapi _testinternalcapi.set_optimizer(self.old_opt) def test_for_loop(self): diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index ccdf3a6cdc0dc7..1b55f1e70b32f5 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -12,6 +12,7 @@ from test import support from test.support import os_helper from test.support.os_helper import TESTFN +from test.support.import_helper import import_module ALL_CJKENCODINGS = [ # _codecs_cn @@ -212,7 +213,7 @@ def test_issue5640(self): @support.cpython_only def test_subinterp(self): # bpo-42846: Test a CJK codec in a subinterpreter - import _testcapi + _testcapi = import_module("_testcapi") encoding = 'cp932' text = "Python の開発は、1990 年ごろから開始されています。" code = textwrap.dedent(""" diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 8829c9a6d88261..f4e954fd02148d 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -5,12 +5,13 @@ import types import unittest from test.support import threading_helper, check_impl_detail, requires_specialization +from test.support.import_helper import import_module # Skip this module on other interpreters, it is cpython specific: if check_impl_detail(cpython=False): raise unittest.SkipTest('implementation detail specific to cpython') -import _testinternalcapi +_testinternalcapi = import_module("_testinternalcapi") def disabling_optimizer(func): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 00b415f43c49b8..094ac13a915e2d 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -57,8 +57,10 @@ except (ImportError, AttributeError): all_users = [] try: + import _testcapi from _testcapi import INT_MAX, PY_SSIZE_T_MAX except ImportError: + _testcapi = None INT_MAX = PY_SSIZE_T_MAX = sys.maxsize try: @@ -5338,6 +5340,7 @@ def test_fork(self): @unittest.skipUnless(sys.platform in ("linux", "android", "darwin"), "Only Linux and macOS detect this today.") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_fork_warns_when_non_python_thread_exists(self): code = """if 1: import os, threading, warnings diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index a17adb89f55360..d4c6fe0124af18 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -2,7 +2,11 @@ import sys import unittest -from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry +try: + from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry +except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") + if sys.platform != 'linux': raise unittest.SkipTest('Linux only') diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 1847ae95db9292..5675db8d1cab6e 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -172,7 +172,10 @@ def test_poll3(self): @cpython_only def test_poll_c_limits(self): - from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX + try: + from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX + except ImportError: + raise unittest.SkipTest("requires _testcapi") pollster = select.poll() pollster.register(1) diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 9d40234ed01697..436fdb38756ddd 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -1372,7 +1372,7 @@ def test_bound_builtin_classmethod_o(self): @support.cpython_only @requires_docstrings def test_module_level_callable_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.func_with_unrepresentable_signature self.assertEqual(self._get_summary_line(builtin), "func_with_unrepresentable_signature(a, b=)") @@ -1382,7 +1382,7 @@ def test_module_level_callable_unrepresentable_default(self): def test_builtin_staticmethod_unrepresentable_default(self): self.assertEqual(self._get_summary_line(str.maketrans), "maketrans(x, y=, z=, /)") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.staticmeth), "staticmeth(a, b=)") @@ -1393,7 +1393,7 @@ def test_unbound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line(dict.pop), "pop(self, key, default=, /) " "unbound builtins.dict method") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.meth), "meth(self, /, a, b=) unbound " @@ -1405,7 +1405,7 @@ def test_bound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line({}.pop), "pop(key, default=, /) " "method of builtins.dict instance") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") obj = _testcapi.DocStringUnrepresentableSignatureTest() self.assertEqual(self._get_summary_line(obj.meth), "meth(a, b=) " @@ -1414,7 +1414,7 @@ def test_bound_builtin_method_unrepresentable_default(self): @support.cpython_only @requires_docstrings def test_unbound_builtin_classmethod_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest descr = cls.__dict__['classmeth'] self.assertEqual(self._get_summary_line(descr), @@ -1424,7 +1424,7 @@ def test_unbound_builtin_classmethod_unrepresentable_default(self): @support.cpython_only @requires_docstrings def test_bound_builtin_classmethod_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.classmeth), "classmeth(a, b=) class method of " diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 6a6b21102fcae8..d222b3803fdba7 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1742,6 +1742,10 @@ def test_other_bug(self): @support.cpython_only def test_uncollectable(self): + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") code = textwrap.dedent(r""" import _testcapi import gc @@ -2124,6 +2128,10 @@ def test_unload_tests(self): def check_add_python_opts(self, option): # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python + try: + import _testinternalcapi + except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") code = textwrap.dedent(r""" import sys import unittest diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index a28d1595f44533..457279a4db687d 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -8,6 +8,7 @@ from test import support from test.support import cpython_only, has_subprocess_support, SuppressCrashReport from test.support.script_helper import kill_python +from test.support.import_helper import import_module if not has_subprocess_support: @@ -64,6 +65,7 @@ class TestInteractiveInterpreter(unittest.TestCase): # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') def test_no_memory(self): + import_module("_testcapi") # Issue #30696: Fix the interactive interpreter looping endlessly when # no memory. Check also that the fix does not break the interactive # loop when an exception is raised. diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 661a859b0d0601..0c4b3bb2ad4d81 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -3,7 +3,6 @@ from test.support import ( is_apple, os_helper, refleak_helper, socket_helper, threading_helper ) - import _thread as thread import array import contextlib @@ -37,6 +36,10 @@ import fcntl except ImportError: fcntl = None +try: + import _testcapi +except ImportError: + _testcapi = None support.requires_working_socket(module=True) @@ -1173,6 +1176,7 @@ def testNtoH(self): self.assertRaises(OverflowError, func, 1<<34) @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testNtoHErrors(self): import _testcapi s_good_values = [0, 1, 2, 0xffff] @@ -1638,6 +1642,7 @@ def testGetaddrinfo(self): except socket.gaierror: pass + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_getaddrinfo_int_port_overflow(self): # gh-74895: Test that getaddrinfo does not raise OverflowError on port. # @@ -1831,6 +1836,7 @@ def test_listen_backlog(self): srv.listen() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_listen_backlog_overflow(self): # Issue 15989 import _testcapi @@ -2712,22 +2718,29 @@ def testDup(self): def _testDup(self): self.serv_conn.send(MSG) - def testShutdown(self): - # Testing shutdown() + def check_shutdown(self): + # Test shutdown() helper msg = self.cli_conn.recv(1024) self.assertEqual(msg, MSG) - # wait for _testShutdown to finish: on OS X, when the server + # wait for _testShutdown[_overflow] to finish: on OS X, when the server # closes the connection the client also becomes disconnected, # and the client's shutdown call will fail. (Issue #4397.) self.done.wait() + def testShutdown(self): + self.check_shutdown() + def _testShutdown(self): self.serv_conn.send(MSG) self.serv_conn.shutdown(2) - testShutdown_overflow = support.cpython_only(testShutdown) + @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def testShutdown_overflow(self): + self.check_shutdown() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def _testShutdown_overflow(self): import _testcapi self.serv_conn.send(MSG) @@ -4884,6 +4897,7 @@ def _testSetBlocking(self): pass @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testSetBlocking_overflow(self): # Issue 15989 import _testcapi diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 588272448bbfda..4182de246a071b 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -34,8 +34,7 @@ is_apple, is_emscripten, is_wasi ) from test.support import gc_collect -from test.support import threading_helper -from _testcapi import INT_MAX, ULLONG_MAX +from test.support import threading_helper, import_helper from os import SEEK_SET, SEEK_CUR, SEEK_END from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath @@ -1202,7 +1201,6 @@ def test_blob_seek_and_tell(self): def test_blob_seek_error(self): msg_oor = "offset out of blob range" msg_orig = "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END" - msg_of = "seek offset results in overflow" dataset = ( (ValueError, msg_oor, lambda: self.blob.seek(1000)), @@ -1214,12 +1212,15 @@ def test_blob_seek_error(self): with self.subTest(exc=exc, msg=msg, fn=fn): self.assertRaisesRegex(exc, msg, fn) + def test_blob_seek_overflow_error(self): # Force overflow errors + msg_of = "seek offset results in overflow" + _testcapi = import_helper.import_module("_testcapi") self.blob.seek(1, SEEK_SET) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_CUR) + self.blob.seek(_testcapi.INT_MAX, SEEK_CUR) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_END) + self.blob.seek(_testcapi.INT_MAX, SEEK_END) def test_blob_read(self): buf = self.blob.read() @@ -1379,14 +1380,17 @@ def test_blob_get_item_error(self): with self.subTest(idx=idx): with self.assertRaisesRegex(IndexError, "index out of range"): self.blob[idx] - with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): - self.blob[ULLONG_MAX] # Provoke read error self.cx.execute("update test set b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob[0] + def test_blob_get_item_error_bigint(self): + _testcapi = import_helper.import_module("_testcapi") + with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): + self.blob[_testcapi.ULLONG_MAX] + def test_blob_set_item_error(self): with self.assertRaisesRegex(TypeError, "cannot be interpreted"): self.blob[0] = b"multiple" diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index d0e4f3c71c15e0..d22698168615e2 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -6,7 +6,10 @@ import sys import unittest from test.support.import_helper import import_module -from _testcapi import get_feature_macros +try: + from _testcapi import get_feature_macros +except ImportError: + raise unittest.SkipTest("requires _testcapi") feature_macros = get_feature_macros() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index f6f23b0afc34c6..6a66df4e897e3f 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1390,7 +1390,7 @@ class SizeofTest(unittest.TestCase): def setUp(self): self.P = struct.calcsize('P') self.longdigit = sys.int_info.sizeof_digit - import _testinternalcapi + _testinternalcapi = import_helper.import_module("_testinternalcapi") self.gc_headsize = _testinternalcapi.SIZEOF_PYGC_HEAD self.managed_pre_header_size = _testinternalcapi.SIZEOF_MANAGED_PRE_HEADER diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 4414d2bb9cdb59..84a946477818df 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -959,6 +959,7 @@ def test_BoundedSemaphore_limit(self): @cpython_only def test_frame_tstate_tracing(self): + _testcapi = import_module("_testcapi") # Issue #14432: Crash when a generator is created in a C thread that is # destroyed while the generator is still used. The issue was that a # generator contains a frame, and the frame kept a reference to the @@ -986,7 +987,6 @@ def callback(): threading.settrace(noop_trace) # Create a generator in a C thread which exits after the call - import _testcapi _testcapi.call_in_temporary_c_thread(callback) # Call the generator in a different Python thread, check that the @@ -1490,6 +1490,7 @@ def task(): @cpython_only def test_daemon_threads_fatal_error(self): + import_module("_testcapi") subinterp_code = f"""if 1: import os import threading @@ -1516,6 +1517,7 @@ def _check_allowed(self, before_start='', *, daemon_allowed=True, daemon=False, ): + import_module("_testinternalcapi") subinterp_code = textwrap.dedent(f""" import test.support import threading diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 17a1a6d0d38d7d..29f5c28e33bb2b 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -35,6 +35,7 @@ def list_test_dirs(self): result.append(line.replace('\\', '').strip()) return result + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") def test_makefile_test_folders(self): test_dirs = self.list_test_dirs() idle_test = 'idlelib/idle_test' diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index d6470fb6919cfd..6fbd292c1e6793 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -14,6 +14,7 @@ from test import support from test.support import script_helper, ALWAYS_EQ from test.support import gc_collect +from test.support import import_helper from test.support import threading_helper # Used in ReferencesTestCase.test_ref_created_during_del() . @@ -161,7 +162,7 @@ def test_basic_callback(self): @support.cpython_only def test_cfunction(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") create_cfunction = _testcapi.create_cfunction f = create_cfunction() wr = weakref.ref(f) diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 95fc4dfbf11e49..8b01c91e0d6bb3 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -275,7 +275,10 @@ def gen_ctypes_test(manifest, args, outfile): import sys import unittest from test.support.import_helper import import_module - from _testcapi import get_feature_macros + try: + from _testcapi import get_feature_macros + except ImportError: + raise unittest.SkipTest("requires _testcapi") feature_macros = get_feature_macros() From fc5f68e58ecfbc8c452e1c2f33a2a53d3f2d7ea2 Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Wed, 3 Apr 2024 07:17:13 -0700 Subject: [PATCH 066/143] gh-59215: unittest: restore _top_level_dir at end of discovery (GH-15242) --- Doc/library/unittest.rst | 15 +++++++---- Lib/test/test_unittest/test_discovery.py | 26 ++++++++++++++++++- Lib/unittest/loader.py | 2 ++ .../2019-08-12-19-08-06.bpo-15010.3bY2CF.rst | 3 +++ 4 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-08-12-19-08-06.bpo-15010.3bY2CF.rst diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index e6140ac70eb87a..3af29f19c802c7 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1880,8 +1880,8 @@ Loading and running tests Python identifiers) will be loaded. All test modules must be importable from the top level of the project. If - the start directory is not the top level directory then the top level - directory must be specified separately. + the start directory is not the top level directory then *top_level_dir* + must be specified separately. If importing a module fails, for example due to a syntax error, then this will be recorded as a single error and discovery will continue. If @@ -1901,9 +1901,11 @@ Loading and running tests package. The pattern is deliberately not stored as a loader attribute so that - packages can continue discovery themselves. *top_level_dir* is stored so - ``load_tests`` does not need to pass this argument in to - ``loader.discover()``. + packages can continue discovery themselves. + + *top_level_dir* is stored internally, and used as a default to any + nested calls to ``discover()``. That is, if a package's ``load_tests`` + calls ``loader.discover()``, it does not need to pass this argument. *start_dir* can be a dotted module name as well as a directory. @@ -1930,6 +1932,9 @@ Loading and running tests *start_dir* can not be a :term:`namespace packages `. It has been broken since Python 3.7 and Python 3.11 officially remove it. + .. versionchanged:: 3.13 + *top_level_dir* is only stored for the duration of *discover* call. + The following attributes of a :class:`TestLoader` can be configured either by subclassing or assignment on an instance: diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index 004898ed431834..a44b18406c08be 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -406,10 +406,34 @@ def _find_tests(start_dir, pattern): top_level_dir = os.path.abspath('/foo/bar') start_dir = os.path.abspath('/foo/bar/baz') self.assertEqual(suite, "['tests']") - self.assertEqual(loader._top_level_dir, top_level_dir) + self.assertEqual(loader._top_level_dir, os.path.abspath('/foo')) self.assertEqual(_find_tests_args, [(start_dir, 'pattern')]) self.assertIn(top_level_dir, sys.path) + def test_discover_should_not_persist_top_level_dir_between_calls(self): + original_isfile = os.path.isfile + original_isdir = os.path.isdir + original_sys_path = sys.path[:] + def restore(): + os.path.isfile = original_isfile + os.path.isdir = original_isdir + sys.path[:] = original_sys_path + self.addCleanup(restore) + + os.path.isfile = lambda path: True + os.path.isdir = lambda path: True + loader = unittest.TestLoader() + loader.suiteClass = str + dir = '/foo/bar' + top_level_dir = '/foo' + + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, None) + + loader._top_level_dir = dir2 = '/previous/dir' + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, dir2) + def test_discover_start_dir_is_package_calls_package_load_tests(self): # This test verifies that the package load_tests in a package is indeed # invoked when the start_dir is a package (and not the top level). diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index 9a3e5cc4bf30e5..22797b83a68bc8 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -254,6 +254,7 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): Paths are sorted before being imported to ensure reproducible execution order even on filesystems with non-alphabetical ordering like ext3/4. """ + original_top_level_dir = self._top_level_dir set_implicit_top = False if top_level_dir is None and self._top_level_dir is not None: # make top_level_dir optional if called from load_tests in a package @@ -307,6 +308,7 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): raise ImportError('Start directory is not importable: %r' % start_dir) tests = list(self._find_tests(start_dir, pattern)) + self._top_level_dir = original_top_level_dir return self.suiteClass(tests) def _get_directory_containing_module(self, module_name): diff --git a/Misc/NEWS.d/next/Library/2019-08-12-19-08-06.bpo-15010.3bY2CF.rst b/Misc/NEWS.d/next/Library/2019-08-12-19-08-06.bpo-15010.3bY2CF.rst new file mode 100644 index 00000000000000..f61a45ed98abad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-08-12-19-08-06.bpo-15010.3bY2CF.rst @@ -0,0 +1,3 @@ +:meth:`unittest.TestLoader.discover` now saves the original value of +``unittest.TestLoader._top_level_dir`` and restores it at the end of the +call. From 03f7aaf953f00bf2953c21a057d8e6e88db659c8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 3 Apr 2024 10:08:18 -0500 Subject: [PATCH 067/143] gh-117215 Make the fromskey() signature match dict.fromkeys(). (gh-117493) --- Lib/collections/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 2a35989ee25a5e..d06d84cbdfcc36 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -1038,9 +1038,9 @@ def __repr__(self): return f'{self.__class__.__name__}({", ".join(map(repr, self.maps))})' @classmethod - def fromkeys(cls, iterable, *args): - 'Create a ChainMap with a single dict created from the iterable.' - return cls(dict.fromkeys(iterable, *args)) + def fromkeys(cls, iterable, value=None, /): + 'Create a new ChainMap with keys from iterable and values set to value.' + return cls(dict.fromkeys(iterable, value)) def copy(self): 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' From 345194de8cb3ceaa40d19353d30ba6e23b6e6edb Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 3 Apr 2024 16:39:40 +0100 Subject: [PATCH 068/143] GH-114847: Raise FileNotFoundError when getcwd() returns '(unreachable)' (#117481) On Linux >= 2.6.36 with glibc < 2.27, `getcwd()` can return a relative pathname starting with '(unreachable)'. We detect this and fail with ENOENT, matching new glibc behaviour. Co-authored-by: Petr Viktorin --- .../2024-04-02-20-30-12.gh-issue-114848.YX4pEc.rst | 2 ++ Modules/posixmodule.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-04-02-20-30-12.gh-issue-114848.YX4pEc.rst diff --git a/Misc/NEWS.d/next/Library/2024-04-02-20-30-12.gh-issue-114848.YX4pEc.rst b/Misc/NEWS.d/next/Library/2024-04-02-20-30-12.gh-issue-114848.YX4pEc.rst new file mode 100644 index 00000000000000..30b1a50976f52d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-02-20-30-12.gh-issue-114848.YX4pEc.rst @@ -0,0 +1,2 @@ +Raise :exc:`FileNotFoundError` when ``getcwd()`` returns '(unreachable)', +which can happen on Linux >= 2.6.36 with glibc < 2.27. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a4b635ef5bf629..fcac3dbe3553ef 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4106,6 +4106,20 @@ posix_getcwd(int use_bytes) else { obj = PyUnicode_DecodeFSDefault(buf); } +#ifdef __linux__ + if (buf[0] != '/') { + /* + * On Linux >= 2.6.36 with glibc < 2.27, getcwd() can return a + * relative pathname starting with '(unreachable)'. We detect this + * and fail with ENOENT, matching newer glibc behaviour. + */ + errno = ENOENT; + path_object_error(obj); + PyMem_RawFree(buf); + return NULL; + } +#endif + assert(buf[0] == '/'); PyMem_RawFree(buf); return obj; From 7ecd55d604a8fa287c1d131cac14d10260be826b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 3 Apr 2024 17:59:18 +0200 Subject: [PATCH 069/143] gh-117431: Adapt str.find and friends to Argument Clinic (#117468) This change gives a significant speedup, as the METH_FASTCALL calling convention is now used. The following methods are adapted: - str.count - str.find - str.index - str.rfind - str.rindex --- Lib/test/string_tests.py | 10 +- ...-04-02-17-37-35.gh-issue-117431.vDKAOn.rst | 12 +- Objects/clinic/unicodeobject.c.h | 281 +++++++++++++- Objects/unicodeobject.c | 353 +++++++----------- 4 files changed, 439 insertions(+), 217 deletions(-) diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 5ade7013328d63..9bb0ce7bb57f8b 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -1503,15 +1503,15 @@ def test_find_etc_raise_correct_error_messages(self): # issue 11828 s = 'hello' x = 'x' - self.assertRaisesRegex(TypeError, r'^find\(', s.find, + self.assertRaisesRegex(TypeError, r'^find\b', s.find, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rfind\(', s.rfind, + self.assertRaisesRegex(TypeError, r'^rfind\b', s.rfind, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^index\(', s.index, + self.assertRaisesRegex(TypeError, r'^index\b', s.index, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rindex\(', s.rindex, + self.assertRaisesRegex(TypeError, r'^rindex\b', s.rindex, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^count\(', s.count, + self.assertRaisesRegex(TypeError, r'^count\b', s.count, x, None, None, None) self.assertRaisesRegex(TypeError, r'^startswith\b', s.startswith, x, None, None, None) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst index 96e14ea0c3b1bd..83f243ae214f7d 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst @@ -1,2 +1,10 @@ -Improve the performance of :meth:`str.startswith` and :meth:`str.endswith` -by adapting them to the :c:macro:`METH_FASTCALL` calling convention. +Improve the performance of the following :class:`str` methods +by adapting them to the :c:macro:`METH_FASTCALL` calling convention: + +* :meth:`~str.count` +* :meth:`~str.endswith` +* :meth:`~str.find` +* :meth:`~str.index` +* :meth:`~str.rfind` +* :meth:`~str.rindex` +* :meth:`~str.startswith` diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 3f6dd8b93a7155..01c40b90d9b4b8 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -136,6 +136,61 @@ unicode_center(PyObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(unicode_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation."); + +#define UNICODE_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(unicode_count), METH_FASTCALL, unicode_count__doc__}, + +static Py_ssize_t +unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_count(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("count", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_count_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_encode__doc__, "encode($self, /, encoding=\'utf-8\', errors=\'strict\')\n" "--\n" @@ -301,6 +356,118 @@ unicode_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(unicode_find__doc__, +"find($self, sub, start=None, end=None, /)\n" +"--\n" +"\n" +"Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Return -1 on failure."); + +#define UNICODE_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(unicode_find), METH_FASTCALL, unicode_find__doc__}, + +static Py_ssize_t +unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_find(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("find", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_find_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_index__doc__, +"index($self, sub, start=None, end=None, /)\n" +"--\n" +"\n" +"Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Raises ValueError when the substring is not found."); + +#define UNICODE_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(unicode_index), METH_FASTCALL, unicode_index__doc__}, + +static Py_ssize_t +unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_index(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("index", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_index_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_isascii__doc__, "isascii($self, /)\n" "--\n" @@ -892,6 +1059,118 @@ unicode_removesuffix(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(unicode_rfind__doc__, +"rfind($self, sub, start=None, end=None, /)\n" +"--\n" +"\n" +"Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Return -1 on failure."); + +#define UNICODE_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(unicode_rfind), METH_FASTCALL, unicode_rfind__doc__}, + +static Py_ssize_t +unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_rfind(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("rfind", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_rfind_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_rindex__doc__, +"rindex($self, sub, start=None, end=None, /)\n" +"--\n" +"\n" +"Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Raises ValueError when the substring is not found."); + +#define UNICODE_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(unicode_rindex), METH_FASTCALL, unicode_rindex__doc__}, + +static Py_ssize_t +unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_rindex(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("rindex", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_rindex_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_rjust__doc__, "rjust($self, width, fillchar=\' \', /)\n" "--\n" @@ -1609,4 +1888,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=1734aa1fcc9b076a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3aa49013ffa3fa93 input=a9049054013a1b77]*/ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index eb83312e9c3a69..e135638c696fa4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -9194,75 +9194,6 @@ _PyUnicode_InsertThousandsGrouping( return count; } -static Py_ssize_t -unicode_count_impl(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end) -{ - assert(PyUnicode_Check(str)); - assert(PyUnicode_Check(substr)); - - Py_ssize_t result; - int kind1, kind2; - const void *buf1 = NULL, *buf2 = NULL; - Py_ssize_t len1, len2; - - kind1 = PyUnicode_KIND(str); - kind2 = PyUnicode_KIND(substr); - if (kind1 < kind2) - return 0; - - len1 = PyUnicode_GET_LENGTH(str); - len2 = PyUnicode_GET_LENGTH(substr); - ADJUST_INDICES(start, end, len1); - if (end - start < len2) - return 0; - - buf1 = PyUnicode_DATA(str); - buf2 = PyUnicode_DATA(substr); - if (kind2 != kind1) { - buf2 = unicode_askind(kind2, buf2, len2, kind1); - if (!buf2) - goto onError; - } - - // We don't reuse `anylib_count` here because of the explicit casts. - switch (kind1) { - case PyUnicode_1BYTE_KIND: - result = ucs1lib_count( - ((const Py_UCS1*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_2BYTE_KIND: - result = ucs2lib_count( - ((const Py_UCS2*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_4BYTE_KIND: - result = ucs4lib_count( - ((const Py_UCS4*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - default: - Py_UNREACHABLE(); - } - - assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); - if (kind2 != kind1) - PyMem_Free((void *)buf2); - - return result; - onError: - assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); - if (kind2 != kind1) - PyMem_Free((void *)buf2); - return -1; -} - Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, @@ -11131,47 +11062,87 @@ PyUnicode_AppendAndDel(PyObject **pleft, PyObject *right) Py_XDECREF(right); } -/* -Wraps asciilib_parse_args_finds() and additionally ensures that the -first argument is a unicode object. -*/ +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +str.count as unicode_count -> Py_ssize_t -static inline int -parse_args_finds_unicode(const char * function_name, PyObject *args, - PyObject **substring, - Py_ssize_t *start, Py_ssize_t *end) -{ - if (asciilib_parse_args_finds(function_name, args, substring, start, end)) { - if (ensure_unicode(*substring) < 0) - return 0; - return 1; - } - return 0; -} + self as str: self + sub as substr: unicode + start: slice_index(accept={int, NoneType}, c_default='0') = None + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + / -PyDoc_STRVAR(count__doc__, - "S.count(sub[, start[, end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of substring sub in\n\ -string S[start:end]. Optional arguments start and end are\n\ -interpreted as in slice notation."); +Return the number of non-overlapping occurrences of substring sub in string S[start:end]. -static PyObject * -unicode_count(PyObject *self, PyObject *args) +Optional arguments start and end are interpreted as in slice notation. +[clinic start generated code]*/ + +static Py_ssize_t +unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=8fcc3aef0b18edbf input=6f168ffd94be8785]*/ { - PyObject *substring = NULL; /* initialize to fix a compiler warning */ - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; + assert(PyUnicode_Check(str)); + assert(PyUnicode_Check(substr)); + Py_ssize_t result; + int kind1, kind2; + const void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; - if (!parse_args_finds_unicode("count", args, &substring, &start, &end)) - return NULL; + kind1 = PyUnicode_KIND(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) + return 0; - result = unicode_count_impl(self, substring, start, end); - if (result == -1) - return NULL; + len1 = PyUnicode_GET_LENGTH(str); + len2 = PyUnicode_GET_LENGTH(substr); + ADJUST_INDICES(start, end, len1); + if (end - start < len2) + return 0; - return PyLong_FromSsize_t(result); + buf1 = PyUnicode_DATA(str); + buf2 = PyUnicode_DATA(substr); + if (kind2 != kind1) { + buf2 = unicode_askind(kind2, buf2, len2, kind1); + if (!buf2) + goto onError; + } + + // We don't reuse `anylib_count` here because of the explicit casts. + switch (kind1) { + case PyUnicode_1BYTE_KIND: + result = ucs1lib_count( + ((const Py_UCS1*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_count( + ((const Py_UCS2*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_count( + ((const Py_UCS4*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + default: + Py_UNREACHABLE(); + } + + assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); + if (kind2 != kind1) + PyMem_Free((void *)buf2); + + return result; + onError: + assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); + if (kind2 != kind1) + PyMem_Free((void *)buf2); + return -1; } /*[clinic input] @@ -11282,33 +11253,25 @@ unicode_expandtabs_impl(PyObject *self, int tabsize) return NULL; } -PyDoc_STRVAR(find__doc__, - "S.find(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - -static PyObject * -unicode_find(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; - - if (!parse_args_finds_unicode("find", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.find as unicode_find = str.count - result = any_find_slice(self, substring, start, end, 1); +Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Return -1 on failure. +[clinic start generated code]*/ - return PyLong_FromSsize_t(result); +static Py_ssize_t +unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=51dbe6255712e278 input=4a89d2d68ef57256]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, 1); + if (result < 0) { + return -1; + } + return result; } static PyObject * @@ -11351,38 +11314,28 @@ unicode_hash(PyObject *self) return x; } -PyDoc_STRVAR(index__doc__, - "S.index(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the substring is not found."); - -static PyObject * -unicode_index(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - Py_ssize_t result; - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - - if (!parse_args_finds_unicode("index", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.index as unicode_index = str.count - result = any_find_slice(self, substring, start, end, 1); +Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Raises ValueError when the substring is not found. +[clinic start generated code]*/ - if (result < 0) { +static Py_ssize_t +unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=77558288837cdf40 input=d986aeac0be14a1c]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, 1); + if (result == -1) { PyErr_SetString(PyExc_ValueError, "substring not found"); - return NULL; } - - return PyLong_FromSsize_t(result); + else if (result < 0) { + return -1; + } + return result; } /*[clinic input] @@ -12462,67 +12415,49 @@ unicode_repr(PyObject *unicode) return repr; } -PyDoc_STRVAR(rfind__doc__, - "S.rfind(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - -static PyObject * -unicode_rfind(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; - - if (!parse_args_finds_unicode("rfind", args, &substring, &start, &end)) - return NULL; - - result = any_find_slice(self, substring, start, end, -1); - - if (result == -2) - return NULL; +/*[clinic input] +str.rfind as unicode_rfind = str.count - return PyLong_FromSsize_t(result); -} +Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -PyDoc_STRVAR(rindex__doc__, - "S.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the substring is not found."); +Optional arguments start and end are interpreted as in slice notation. +Return -1 on failure. +[clinic start generated code]*/ -static PyObject * -unicode_rindex(PyObject *self, PyObject *args) +static Py_ssize_t +unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=880b29f01dd014c8 input=898361fb71f59294]*/ { - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; + Py_ssize_t result = any_find_slice(str, substr, start, end, -1); + if (result < 0) { + return -1; + } + return result; +} - if (!parse_args_finds_unicode("rindex", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.rindex as unicode_rindex = str.count - result = any_find_slice(self, substring, start, end, -1); +Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Raises ValueError when the substring is not found. +[clinic start generated code]*/ - if (result < 0) { +static Py_ssize_t +unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=5f3aef124c867fe1 input=35943dead6c1ea9d]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, -1); + if (result == -1) { PyErr_SetString(PyExc_ValueError, "substring not found"); - return NULL; } - - return PyLong_FromSsize_t(result); + else if (result < 0) { + return -1; + } + return result; } /*[clinic input] @@ -13562,16 +13497,16 @@ static PyMethodDef unicode_methods[] = { UNICODE_CASEFOLD_METHODDEF UNICODE_TITLE_METHODDEF UNICODE_CENTER_METHODDEF - {"count", (PyCFunction) unicode_count, METH_VARARGS, count__doc__}, + UNICODE_COUNT_METHODDEF UNICODE_EXPANDTABS_METHODDEF - {"find", (PyCFunction) unicode_find, METH_VARARGS, find__doc__}, + UNICODE_FIND_METHODDEF UNICODE_PARTITION_METHODDEF - {"index", (PyCFunction) unicode_index, METH_VARARGS, index__doc__}, + UNICODE_INDEX_METHODDEF UNICODE_LJUST_METHODDEF UNICODE_LOWER_METHODDEF UNICODE_LSTRIP_METHODDEF - {"rfind", (PyCFunction) unicode_rfind, METH_VARARGS, rfind__doc__}, - {"rindex", (PyCFunction) unicode_rindex, METH_VARARGS, rindex__doc__}, + UNICODE_RFIND_METHODDEF + UNICODE_RINDEX_METHODDEF UNICODE_RJUST_METHODDEF UNICODE_RSTRIP_METHODDEF UNICODE_RPARTITION_METHODDEF From 976bcb2379709da57073a9e07b518ff51daa617a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 3 Apr 2024 10:58:39 -0600 Subject: [PATCH 070/143] gh-76785: Raise InterpreterError, Not RuntimeError (gh-117489) I had meant to switch everything to InterpreterError when I added it a while back. At the time I missed a few key spots. As part of this, I've added print-the-exception to _PyXI_InitTypes() and fixed an error case in `_PyStaticType_InitBuiltin(). --- Lib/test/support/interpreters/__init__.py | 4 ++-- Lib/test/test__xxsubinterpreters.py | 12 ++++++------ Lib/test/test_interpreters/test_api.py | 10 +++++----- Modules/_xxsubinterpretersmodule.c | 8 ++++---- Objects/typeobject.c | 1 + Python/crossinterp.c | 9 +++++---- Python/crossinterp_exceptions.h | 17 ++++++++++++++++- Python/pystate.c | 2 +- 8 files changed, 40 insertions(+), 23 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 8316b5e4a93bb6..8be4ee736aa93b 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -43,7 +43,7 @@ def __getattr__(name): {formatted} """.strip() -class ExecutionFailed(RuntimeError): +class ExecutionFailed(InterpreterError): """An unhandled exception happened during execution. This is raised from Interpreter.exec() and Interpreter.call(). @@ -158,7 +158,7 @@ def close(self): """Finalize and destroy the interpreter. Attempting to destroy the current interpreter results - in a RuntimeError. + in an InterpreterError. """ return _interpreters.destroy(self._id) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index f674771c27cbb1..841077adbb0f16 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -80,7 +80,7 @@ def clean_up_interpreters(): continue try: interpreters.destroy(id) - except RuntimeError: + except interpreters.InterpreterError: pass # already destroyed @@ -464,11 +464,11 @@ def test_all(self): def test_main(self): main, = interpreters.list_all() - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.InterpreterError): interpreters.destroy(main) def f(): - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.InterpreterError): interpreters.destroy(main) t = threading.Thread(target=f) @@ -496,7 +496,7 @@ def test_from_current(self): import _xxsubinterpreters as _interpreters try: _interpreters.destroy({id}) - except RuntimeError: + except interpreters.InterpreterError: pass """) @@ -531,7 +531,7 @@ def test_still_running(self): self.assertTrue(interpreters.is_running(interp), msg=f"Interp {interp} should be running before destruction.") - with self.assertRaises(RuntimeError, + with self.assertRaises(interpreters.InterpreterError, msg=f"Should not be able to destroy interp {interp} while it's still running."): interpreters.destroy(interp) self.assertTrue(interpreters.is_running(interp)) @@ -676,7 +676,7 @@ def test_fork(self): def test_already_running(self): with _running(self.id): - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.InterpreterError): interpreters.run_string(self.id, 'print("spam")') def test_does_not_exist(self): diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 2aa7f9bb61aa5b..a326b39fd234c7 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -364,11 +364,11 @@ def test_all(self): def test_main(self): main, = interpreters.list_all() - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.InterpreterError): main.close() def f(): - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.InterpreterError): main.close() t = threading.Thread(target=f) @@ -389,7 +389,7 @@ def test_from_current(self): interp = interpreters.Interpreter({interp.id}) try: interp.close() - except RuntimeError: + except interpreters.InterpreterError: print('failed') """)) self.assertEqual(out.strip(), 'failed') @@ -424,7 +424,7 @@ def test_still_running(self): main, = interpreters.list_all() interp = interpreters.create() with _running(interp): - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.InterpreterError): interp.close() self.assertTrue(interp.is_running()) @@ -1103,7 +1103,7 @@ def test_create(self): self.assert_ns_equal(config, default) with self.subTest('arg: \'empty\''): - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.InterpreterError): # The "empty" config isn't viable on its own. _interpreters.create('empty') diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 9c2774e4f82def..94b8ee35001732 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -612,7 +612,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) // XXX Move the chained exception to interpreters.create()? PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); - PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); + PyErr_SetString(PyExc_InterpreterError, "interpreter creation failed"); _PyErr_ChainExceptions1(exc); return NULL; } @@ -664,7 +664,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } if (interp == current) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_InterpreterError, "cannot destroy the current interpreter"); return NULL; } @@ -673,7 +673,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) /* XXX We *could* support destroying a running interpreter but aren't going to worry about it for now. */ if (is_running_main(interp)) { - PyErr_Format(PyExc_RuntimeError, "interpreter running"); + PyErr_Format(PyExc_InterpreterError, "interpreter running"); return NULL; } @@ -693,7 +693,7 @@ PyDoc_STRVAR(destroy_doc, \n\ Destroy the identified interpreter.\n\ \n\ -Attempting to destroy the current interpreter results in a RuntimeError.\n\ +Attempting to destroy the current interpreter raises InterpreterError.\n\ So does an unrecognized ID."); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 6df9986b82e77c..51ceb7d7de1cb6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7963,6 +7963,7 @@ _PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) res = type_ready(self, !ismain); END_TYPE_LOCK() if (res < 0) { + _PyStaticType_ClearWeakRefs(interp, self); static_builtin_state_clear(interp, self); } return res; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 18dec4dd959044..16efe9c3958f87 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -845,7 +845,7 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) return 0; case _PyXI_ERR_OTHER: // XXX msg? - PyErr_SetNone(PyExc_RuntimeError); + PyErr_SetNone(PyExc_InterpreterError); break; case _PyXI_ERR_NO_MEMORY: PyErr_NoMemory(); @@ -856,11 +856,11 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) _PyInterpreterState_FailIfRunningMain(interp); break; case _PyXI_ERR_MAIN_NS_FAILURE: - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_InterpreterError, "failed to get __main__ namespace"); break; case _PyXI_ERR_APPLY_NS_FAILURE: - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_InterpreterError, "failed to apply namespace to __main__"); break; case _PyXI_ERR_NOT_SHAREABLE: @@ -935,7 +935,7 @@ _PyXI_ApplyError(_PyXI_error *error) if (error->uncaught.type.name != NULL || error->uncaught.msg != NULL) { // __context__ will be set to a proxy of the propagated exception. PyObject *exc = PyErr_GetRaisedException(); - _PyXI_excinfo_Apply(&error->uncaught, PyExc_RuntimeError); + _PyXI_excinfo_Apply(&error->uncaught, PyExc_InterpreterError); PyObject *exc2 = PyErr_GetRaisedException(); PyException_SetContext(exc, exc2); PyErr_SetRaisedException(exc); @@ -1671,6 +1671,7 @@ PyStatus _PyXI_InitTypes(PyInterpreterState *interp) { if (init_exceptions(interp) < 0) { + PyErr_PrintEx(0); return _PyStatus_ERR("failed to initialize an exception type"); } return _PyStatus_OK(); diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index e418cf91d4a7af..0f324bac48a2d8 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -5,6 +5,9 @@ static PyTypeObject _PyExc_InterpreterError = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "interpreters.InterpreterError", .tp_doc = PyDoc_STR("A cross-interpreter operation failed"), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + //.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse, + //.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear, //.tp_base = (PyTypeObject *)PyExc_BaseException, }; PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; @@ -15,6 +18,9 @@ static PyTypeObject _PyExc_InterpreterNotFoundError = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "interpreters.InterpreterNotFoundError", .tp_doc = PyDoc_STR("An interpreter was not found"), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + //.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse, + //.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear, .tp_base = &_PyExc_InterpreterError, }; PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; @@ -55,16 +61,25 @@ _get_not_shareable_error_type(PyInterpreterState *interp) static int init_exceptions(PyInterpreterState *interp) { + PyTypeObject *base = (PyTypeObject *)PyExc_BaseException; + // builtin static types - _PyExc_InterpreterError.tp_base = (PyTypeObject *)PyExc_BaseException; + + _PyExc_InterpreterError.tp_base = base; + _PyExc_InterpreterError.tp_traverse = base->tp_traverse; + _PyExc_InterpreterError.tp_clear = base->tp_clear; if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { return -1; } + + _PyExc_InterpreterNotFoundError.tp_traverse = base->tp_traverse; + _PyExc_InterpreterNotFoundError.tp_clear = base->tp_clear; if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { return -1; } // heap types + // We would call _init_not_shareable_error_type() here too, // but that leads to ref leaks diff --git a/Python/pystate.c b/Python/pystate.c index 925d1cff866f18..892e740493cdfd 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1073,7 +1073,7 @@ int _PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) { if (interp->threads.main != NULL) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_InterpreterError, "interpreter already running"); return -1; } From 1c434688866db79082def4f9ef572b85d85908c8 Mon Sep 17 00:00:00 2001 From: Peter Lazorchak Date: Wed, 3 Apr 2024 10:14:18 -0700 Subject: [PATCH 071/143] gh-116168: Remove extra `_CHECK_STACK_SPACE` uops (#117242) This merges all `_CHECK_STACK_SPACE` uops in a trace into a single `_CHECK_STACK_SPACE_OPERAND` uop that checks whether there is enough stack space for all calls included in the entire trace. --- Include/internal/pycore_uop_ids.h | 215 ++++++++++---------- Include/internal/pycore_uop_metadata.h | 4 + Lib/test/test_capi/test_opt.py | 268 ++++++++++++++++++++++++- Modules/_testinternalcapi.c | 12 ++ Python/bytecodes.c | 6 + Python/executor_cases.c.h | 8 + Python/optimizer_analysis.c | 103 ++++++++-- Python/optimizer_cases.c.h | 4 + 8 files changed, 494 insertions(+), 126 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 54dc6dcf408116..3e4dd8b4009cd4 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -68,14 +68,15 @@ extern "C" { #define _CHECK_PEP_523 330 #define _CHECK_PERIODIC 331 #define _CHECK_STACK_SPACE 332 -#define _CHECK_VALIDITY 333 -#define _CHECK_VALIDITY_AND_SET_IP 334 -#define _COLD_EXIT 335 -#define _COMPARE_OP 336 -#define _COMPARE_OP_FLOAT 337 -#define _COMPARE_OP_INT 338 -#define _COMPARE_OP_STR 339 -#define _CONTAINS_OP 340 +#define _CHECK_STACK_SPACE_OPERAND 333 +#define _CHECK_VALIDITY 334 +#define _CHECK_VALIDITY_AND_SET_IP 335 +#define _COLD_EXIT 336 +#define _COMPARE_OP 337 +#define _COMPARE_OP_FLOAT 338 +#define _COMPARE_OP_INT 339 +#define _COMPARE_OP_STR 340 +#define _CONTAINS_OP 341 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -87,47 +88,47 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 341 +#define _DEOPT 342 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE #define _END_SEND END_SEND -#define _ERROR_POP_N 342 +#define _ERROR_POP_N 343 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 343 +#define _FATAL_ERROR 344 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 344 +#define _FOR_ITER 345 #define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 345 +#define _FOR_ITER_TIER_TWO 346 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 346 -#define _GUARD_BOTH_INT 347 -#define _GUARD_BOTH_UNICODE 348 -#define _GUARD_BUILTINS_VERSION 349 -#define _GUARD_DORV_NO_DICT 350 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351 -#define _GUARD_GLOBALS_VERSION 352 -#define _GUARD_IS_FALSE_POP 353 -#define _GUARD_IS_NONE_POP 354 -#define _GUARD_IS_NOT_NONE_POP 355 -#define _GUARD_IS_TRUE_POP 356 -#define _GUARD_KEYS_VERSION 357 -#define _GUARD_NOT_EXHAUSTED_LIST 358 -#define _GUARD_NOT_EXHAUSTED_RANGE 359 -#define _GUARD_NOT_EXHAUSTED_TUPLE 360 -#define _GUARD_TYPE_VERSION 361 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 362 -#define _INIT_CALL_PY_EXACT_ARGS 363 -#define _INIT_CALL_PY_EXACT_ARGS_0 364 -#define _INIT_CALL_PY_EXACT_ARGS_1 365 -#define _INIT_CALL_PY_EXACT_ARGS_2 366 -#define _INIT_CALL_PY_EXACT_ARGS_3 367 -#define _INIT_CALL_PY_EXACT_ARGS_4 368 +#define _GUARD_BOTH_FLOAT 347 +#define _GUARD_BOTH_INT 348 +#define _GUARD_BOTH_UNICODE 349 +#define _GUARD_BUILTINS_VERSION 350 +#define _GUARD_DORV_NO_DICT 351 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 352 +#define _GUARD_GLOBALS_VERSION 353 +#define _GUARD_IS_FALSE_POP 354 +#define _GUARD_IS_NONE_POP 355 +#define _GUARD_IS_NOT_NONE_POP 356 +#define _GUARD_IS_TRUE_POP 357 +#define _GUARD_KEYS_VERSION 358 +#define _GUARD_NOT_EXHAUSTED_LIST 359 +#define _GUARD_NOT_EXHAUSTED_RANGE 360 +#define _GUARD_NOT_EXHAUSTED_TUPLE 361 +#define _GUARD_TYPE_VERSION 362 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 363 +#define _INIT_CALL_PY_EXACT_ARGS 364 +#define _INIT_CALL_PY_EXACT_ARGS_0 365 +#define _INIT_CALL_PY_EXACT_ARGS_1 366 +#define _INIT_CALL_PY_EXACT_ARGS_2 367 +#define _INIT_CALL_PY_EXACT_ARGS_3 368 +#define _INIT_CALL_PY_EXACT_ARGS_4 369 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -144,65 +145,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 369 -#define _IS_NONE 370 +#define _INTERNAL_INCREMENT_OPT_COUNTER 370 +#define _IS_NONE 371 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 371 -#define _ITER_CHECK_RANGE 372 -#define _ITER_CHECK_TUPLE 373 -#define _ITER_JUMP_LIST 374 -#define _ITER_JUMP_RANGE 375 -#define _ITER_JUMP_TUPLE 376 -#define _ITER_NEXT_LIST 377 -#define _ITER_NEXT_RANGE 378 -#define _ITER_NEXT_TUPLE 379 -#define _JUMP_TO_TOP 380 +#define _ITER_CHECK_LIST 372 +#define _ITER_CHECK_RANGE 373 +#define _ITER_CHECK_TUPLE 374 +#define _ITER_JUMP_LIST 375 +#define _ITER_JUMP_RANGE 376 +#define _ITER_JUMP_TUPLE 377 +#define _ITER_NEXT_LIST 378 +#define _ITER_NEXT_RANGE 379 +#define _ITER_NEXT_TUPLE 380 +#define _JUMP_TO_TOP 381 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 381 -#define _LOAD_ATTR_CLASS 382 -#define _LOAD_ATTR_CLASS_0 383 -#define _LOAD_ATTR_CLASS_1 384 +#define _LOAD_ATTR 382 +#define _LOAD_ATTR_CLASS 383 +#define _LOAD_ATTR_CLASS_0 384 +#define _LOAD_ATTR_CLASS_1 385 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 385 -#define _LOAD_ATTR_INSTANCE_VALUE_0 386 -#define _LOAD_ATTR_INSTANCE_VALUE_1 387 -#define _LOAD_ATTR_METHOD_LAZY_DICT 388 -#define _LOAD_ATTR_METHOD_NO_DICT 389 -#define _LOAD_ATTR_METHOD_WITH_VALUES 390 -#define _LOAD_ATTR_MODULE 391 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 392 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 393 +#define _LOAD_ATTR_INSTANCE_VALUE 386 +#define _LOAD_ATTR_INSTANCE_VALUE_0 387 +#define _LOAD_ATTR_INSTANCE_VALUE_1 388 +#define _LOAD_ATTR_METHOD_LAZY_DICT 389 +#define _LOAD_ATTR_METHOD_NO_DICT 390 +#define _LOAD_ATTR_METHOD_WITH_VALUES 391 +#define _LOAD_ATTR_MODULE 392 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 393 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 394 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 394 -#define _LOAD_ATTR_SLOT_0 395 -#define _LOAD_ATTR_SLOT_1 396 -#define _LOAD_ATTR_WITH_HINT 397 +#define _LOAD_ATTR_SLOT 395 +#define _LOAD_ATTR_SLOT_0 396 +#define _LOAD_ATTR_SLOT_1 397 +#define _LOAD_ATTR_WITH_HINT 398 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 398 -#define _LOAD_CONST_INLINE_BORROW 399 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 400 -#define _LOAD_CONST_INLINE_WITH_NULL 401 +#define _LOAD_CONST_INLINE 399 +#define _LOAD_CONST_INLINE_BORROW 400 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 401 +#define _LOAD_CONST_INLINE_WITH_NULL 402 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 402 -#define _LOAD_FAST_0 403 -#define _LOAD_FAST_1 404 -#define _LOAD_FAST_2 405 -#define _LOAD_FAST_3 406 -#define _LOAD_FAST_4 407 -#define _LOAD_FAST_5 408 -#define _LOAD_FAST_6 409 -#define _LOAD_FAST_7 410 +#define _LOAD_FAST 403 +#define _LOAD_FAST_0 404 +#define _LOAD_FAST_1 405 +#define _LOAD_FAST_2 406 +#define _LOAD_FAST_3 407 +#define _LOAD_FAST_4 408 +#define _LOAD_FAST_5 409 +#define _LOAD_FAST_6 410 +#define _LOAD_FAST_7 411 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 411 -#define _LOAD_GLOBAL_BUILTINS 412 -#define _LOAD_GLOBAL_MODULE 413 +#define _LOAD_GLOBAL 412 +#define _LOAD_GLOBAL_BUILTINS 413 +#define _LOAD_GLOBAL_MODULE 414 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -216,49 +217,49 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 414 -#define _POP_JUMP_IF_FALSE 415 -#define _POP_JUMP_IF_TRUE 416 +#define _POP_FRAME 415 +#define _POP_JUMP_IF_FALSE 416 +#define _POP_JUMP_IF_TRUE 417 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 417 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 418 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 418 +#define _PUSH_FRAME 419 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 419 +#define _REPLACE_WITH_TRUE 420 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 420 -#define _SEND 421 +#define _SAVE_RETURN_OFFSET 421 +#define _SEND 422 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SIDE_EXIT 422 -#define _START_EXECUTOR 423 -#define _STORE_ATTR 424 -#define _STORE_ATTR_INSTANCE_VALUE 425 -#define _STORE_ATTR_SLOT 426 +#define _SIDE_EXIT 423 +#define _START_EXECUTOR 424 +#define _STORE_ATTR 425 +#define _STORE_ATTR_INSTANCE_VALUE 426 +#define _STORE_ATTR_SLOT 427 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 427 -#define _STORE_FAST_0 428 -#define _STORE_FAST_1 429 -#define _STORE_FAST_2 430 -#define _STORE_FAST_3 431 -#define _STORE_FAST_4 432 -#define _STORE_FAST_5 433 -#define _STORE_FAST_6 434 -#define _STORE_FAST_7 435 +#define _STORE_FAST 428 +#define _STORE_FAST_0 429 +#define _STORE_FAST_1 430 +#define _STORE_FAST_2 431 +#define _STORE_FAST_3 432 +#define _STORE_FAST_4 433 +#define _STORE_FAST_5 434 +#define _STORE_FAST_6 435 +#define _STORE_FAST_7 436 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 436 +#define _STORE_SUBSCR 437 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 437 +#define _TO_BOOL 438 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -268,12 +269,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 438 +#define _UNPACK_SEQUENCE 439 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 438 +#define MAX_UOP_ID 439 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0f2046fb3d0c3d..111824a938f6cc 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -228,6 +228,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG, [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, [_SET_IP] = 0, + [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_EXIT_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, @@ -302,6 +303,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", [_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP", [_COLD_EXIT] = "_COLD_EXIT", @@ -902,6 +904,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _SET_IP: return 0; + case _CHECK_STACK_SPACE_OPERAND: + return 0; case _SAVE_RETURN_OFFSET: return 0; case _EXIT_TRACE: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index b59f4b74a8593e..ceb49c3c7129cb 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -952,6 +952,269 @@ def testfunc(n): _, ex = self._run_with_optimizer(testfunc, 16) self.assertIsNone(ex) + def test_combine_stack_space_checks_sequential(self): + def dummy12(x): + return x - 1 + def dummy13(y): + z = y + 2 + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(7) + c, d = dummy13(9) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 832) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_POP_FRAME"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # sequential calls: max(12, 13) == 13 + largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_nested(self): + def dummy12(x): + return x + 3 + def dummy15(y): + z = dummy12(y) + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b, c = dummy15(2) + a += b + c + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 224) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_POP_FRAME"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # nested calls: 15 + 12 == 27 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy12.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls(self): + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(5) + c, d, e = dummy18(2) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 800) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_POP_FRAME"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(12, 18 + max(12, 13)) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls_different_order(self): + # same as `several_calls` but with top-level calls reversed + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + c, d, e = dummy18(2) + b = dummy12(5) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 800) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_POP_FRAME"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(18 + max(12, 13), 12) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_complex(self): + def dummy0(x): + return x + def dummy1(x): + return dummy0(x) + def dummy2(x): + return dummy1(x) + def dummy3(x): + return dummy0(x) + def dummy4(x): + y = dummy0(x) + return dummy3(y) + def dummy5(x): + return dummy2(x) + def dummy6(x): + y = dummy5(x) + z = dummy0(y) + return dummy4(z) + def testfunc(n): + a = 0; + for _ in range(32): + b = dummy5(1) + c = dummy0(1) + d = dummy6(1) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 96) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 15) + self.assertEqual(uop_names.count("_POP_FRAME"), 15) + + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy6.__code__) + + _testinternalcapi.get_co_framesize(dummy5.__code__) + + _testinternalcapi.get_co_framesize(dummy2.__code__) + + _testinternalcapi.get_co_framesize(dummy1.__code__) + + _testinternalcapi.get_co_framesize(dummy0.__code__) + ) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_large_framesize(self): + # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is + # actually doing its job. Note that the resulting trace hits + # UOP_MAX_TRACE_LENGTH, but since all _CHECK_STACK_SPACEs happen early, this + # test is still meaningful. + repetitions = 10000 + ns = {} + header = """ + def dummy_large(a0): + """ + body = "".join([f""" + a{n+1} = a{n} + 1 + """ for n in range(repetitions)]) + return_ = f""" + return a{repetitions-1} + """ + exec(textwrap.dedent(header + body + return_), ns, ns) + dummy_large = ns['dummy_large'] + + # this is something like: + # + # def dummy_large(a0): + # a1 = a0 + 1 + # a2 = a1 + 1 + # .... + # a9999 = a9998 + 1 + # return a9999 + + def dummy15(z): + y = dummy_large(z) + return y + 3 + + def testfunc(n): + b = 0 + for _ in range(n): + b += dummy15(7) + return b + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 32 * (repetitions + 9)) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + + # this hits a different case during trace projection in refcount test runs only, + # so we need to account for both possibilities + self.assertIn(uop_names.count("_CHECK_STACK_SPACE"), [0, 1]) + if uop_names.count("_CHECK_STACK_SPACE") == 0: + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy_large.__code__) + ) + else: + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_recursion(self): + def dummy15(x): + while x > 0: + return dummy15(x - 1) + return 42 + def testfunc(n): + a = 0 + for _ in range(n): + a += dummy15(n) + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 42 * 32) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_POP_FRAME"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 1) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + def test_many_nested(self): # overflow the trace_stack def dummy_a(x): @@ -976,8 +1239,9 @@ def testfunc(n): a += dummy_h(n) return a - self._run_with_optimizer(testfunc, 32) - + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 32 * 32) + self.assertIsNone(ex) if __name__ == "__main__": unittest.main() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c5d65a373906f2..6b5d99f6ffac1f 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -959,6 +959,17 @@ iframe_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(PyUnstable_InterpreterFrame_GetLasti(f)); } +static PyObject * +get_co_framesize(PyObject *self, PyObject *arg) +{ + if (!PyCode_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "argument must be a code object"); + return NULL; + } + PyCodeObject *code = (PyCodeObject *)arg; + return PyLong_FromLong(code->co_framesize); +} + static PyObject * new_counter_optimizer(PyObject *self, PyObject *arg) { @@ -1715,6 +1726,7 @@ static PyMethodDef module_functions[] = { {"iframe_getcode", iframe_getcode, METH_O, NULL}, {"iframe_getline", iframe_getline, METH_O, NULL}, {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, + {"get_co_framesize", get_co_framesize, METH_O, NULL}, {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ce208aac9c7953..fa53c969fe361e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4094,6 +4094,12 @@ dummy_func( frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; } + tier2 op(_CHECK_STACK_SPACE_OPERAND, (framesize/2 --)) { + assert(framesize <= INT_MAX); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, framesize)); + DEOPT_IF(tstate->py_recursion_remaining <= 1); + } + op(_SAVE_RETURN_OFFSET, (--)) { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 82f2171f1ede83..98476798fbbbdf 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3592,6 +3592,14 @@ break; } + case _CHECK_STACK_SPACE_OPERAND: { + uint32_t framesize = (uint32_t)CURRENT_OPERAND(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) JUMP_TO_JUMP_TARGET(); + if (tstate->py_recursion_remaining <= 1) JUMP_TO_JUMP_TARGET(); + break; + } + case _SAVE_RETURN_OFFSET: { oparg = CURRENT_OPARG(); #if TIER_ONE diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6f553f8ab8ad2e..a21679f366a74e 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -529,14 +529,41 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } } } - Py_FatalError("No terminating instruction"); Py_UNREACHABLE(); } +/* _PUSH_FRAME/_POP_FRAME's operand can be 0, a PyFunctionObject *, or a + * PyCodeObject *. Retrieve the code object if possible. + */ +static PyCodeObject * +get_co(_PyUOpInstruction *op) +{ + assert(op->opcode == _PUSH_FRAME || op->opcode == _POP_FRAME); + PyCodeObject *co = NULL; + uint64_t operand = op->operand; + if (operand == 0) { + return NULL; + } + if (operand & 1) { + co = (PyCodeObject *)(operand & ~1); + } + else { + PyFunctionObject *func = (PyFunctionObject *)operand; + assert(PyFunction_Check(func)); + co = (PyCodeObject *)func->func_code; + } + assert(PyCode_Check(co)); + return co; +} + static void peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size) { PyCodeObject *co = _PyFrame_GetCode(frame); + int curr_space = 0; + int max_space = 0; + _PyUOpInstruction *first_valid_check_stack = NULL; + _PyUOpInstruction *corresponding_check_stack = NULL; for (int pc = 0; pc < buffer_size; pc++) { int opcode = buffer[pc].opcode; switch(opcode) { @@ -547,8 +574,7 @@ peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_s buffer[pc].operand = (uintptr_t)val; break; } - case _CHECK_PEP_523: - { + case _CHECK_PEP_523: { /* Setting the eval frame function invalidates * all executors, so no need to check dynamically */ if (_PyInterpreterState_GET()->eval_frame == NULL) { @@ -556,29 +582,72 @@ peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_s } break; } - case _PUSH_FRAME: - case _POP_FRAME: - { - uint64_t operand = buffer[pc].operand; - if (operand & 1) { - co = (PyCodeObject *)(operand & ~1); - assert(PyCode_Check(co)); - } - else if (operand == 0) { - co = NULL; + case _CHECK_STACK_SPACE: { + assert(corresponding_check_stack == NULL); + corresponding_check_stack = &buffer[pc]; + break; + } + case _PUSH_FRAME: { + assert(corresponding_check_stack != NULL); + co = get_co(&buffer[pc]); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + goto finish; + } + int framesize = co->co_framesize; + assert(framesize > 0); + curr_space += framesize; + if (curr_space < 0 || curr_space > INT32_MAX) { + // won't fit in signed 32-bit int + goto finish; + } + max_space = curr_space > max_space ? curr_space : max_space; + if (first_valid_check_stack == NULL) { + first_valid_check_stack = corresponding_check_stack; } else { - PyFunctionObject *func = (PyFunctionObject *)operand; - assert(PyFunction_Check(func)); - co = (PyCodeObject *)func->func_code; + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } + corresponding_check_stack = NULL; + break; + } + case _POP_FRAME: { + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_co(&buffer[pc]); + if (co == NULL) { + // might be impossible, but bailing is still safe + goto finish; } break; } case _JUMP_TO_TOP: case _EXIT_TRACE: - return; + goto finish; +#ifdef Py_DEBUG + case _CHECK_STACK_SPACE_OPERAND: { + /* We should never see _CHECK_STACK_SPACE_OPERANDs. + * They are only created at the end of this pass. */ + Py_UNREACHABLE(); + } +#endif } } + Py_UNREACHABLE(); +finish: + if (first_valid_check_stack != NULL) { + assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); + assert(max_space > 0); + assert(max_space <= INT_MAX); + assert(max_space <= INT32_MAX); + first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; + first_valid_check_stack->operand = max_space; + } } // 0 - failure, no error raised, just fall back to Tier 1 diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b4a1da8aec14af..209be370c4aa38 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1906,6 +1906,10 @@ break; } + case _CHECK_STACK_SPACE_OPERAND: { + break; + } + case _SAVE_RETURN_OFFSET: { break; } From c43f6a4dfaa1c1974141e2f7014a7f98ca3a3f93 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 3 Apr 2024 20:17:51 +0200 Subject: [PATCH 072/143] gh-113317: Argument Clinic: Add libclinic.clanguage (#117455) Add libclinic.clanguage module and move the following classes and functions there: * CLanguage * declare_parser() Add libclinic.codegen and move the following classes there: * BlockPrinter * BufferSeries * Destination Move the following functions to libclinic.function: * permute_left_option_groups() * permute_optional_groups() * permute_right_option_groups() --- Lib/test/test_clinic.py | 13 +- Tools/clinic/clinic.py | 1602 +-------------------------- Tools/clinic/libclinic/clanguage.py | 1364 +++++++++++++++++++++++ Tools/clinic/libclinic/codegen.py | 187 ++++ Tools/clinic/libclinic/function.py | 71 ++ 5 files changed, 1635 insertions(+), 1602 deletions(-) create mode 100644 Tools/clinic/libclinic/clanguage.py create mode 100644 Tools/clinic/libclinic/codegen.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index a07641d1dfda54..df8b3d261c4278 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -18,6 +18,9 @@ with test_tools.imports_under_tool('clinic'): import libclinic from libclinic.converters import int_converter, str_converter + from libclinic.function import ( + permute_optional_groups, permute_right_option_groups, + permute_left_option_groups) import clinic from clinic import DSLParser @@ -679,7 +682,7 @@ def test_parse_file_strange_extension(self) -> None: class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): - computed = clinic.permute_optional_groups(l, m, r) + computed = permute_optional_groups(l, m, r) self.assertEqual(output, computed) def test_range(self): @@ -721,7 +724,7 @@ def test_right_only(self): def test_have_left_options_but_required_is_empty(self): def fn(): - clinic.permute_optional_groups(['a'], [], []) + permute_optional_groups(['a'], [], []) self.assertRaises(ValueError, fn) @@ -3764,7 +3767,7 @@ def test_permute_left_option_groups(self): (1, 2, 3), ) data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. - actual = tuple(clinic.permute_left_option_groups(data)) + actual = tuple(permute_left_option_groups(data)) self.assertEqual(actual, expected) def test_permute_right_option_groups(self): @@ -3775,7 +3778,7 @@ def test_permute_right_option_groups(self): (1, 2, 3), ) data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. - actual = tuple(clinic.permute_right_option_groups(data)) + actual = tuple(permute_right_option_groups(data)) self.assertEqual(actual, expected) def test_permute_optional_groups(self): @@ -3854,7 +3857,7 @@ def test_permute_optional_groups(self): for params in dataset: with self.subTest(**params): left, required, right, expected = params.values() - permutations = clinic.permute_optional_groups(left, required, right) + permutations = permute_optional_groups(left, required, right) actual = tuple(permutations) self.assertEqual(actual, expected) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 97b1f46a13411b..d8043128bf3d01 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -9,31 +9,23 @@ import argparse import ast import contextlib -import dataclasses as dc import enum import functools import inspect import io -import itertools import os import pprint import re import shlex import sys -import textwrap from collections.abc import ( Callable, - Iterable, - Iterator, Sequence, ) -from operator import attrgetter from types import FunctionType, NoneType from typing import ( Any, - Final, - Literal, NamedTuple, NoReturn, Protocol, @@ -44,7 +36,7 @@ import libclinic import libclinic.cpp from libclinic import ( - ClinicError, Sentinels, VersionTuple, + ClinicError, VersionTuple, fail, warn, unspecified, unknown, NULL) from libclinic.function import ( Module, Class, Function, Parameter, @@ -53,16 +45,18 @@ GETTER, SETTER) from libclinic.language import Language, PythonLanguage from libclinic.block_parser import Block, BlockParser -from libclinic.crenderdata import CRenderData, Include, TemplateDict +from libclinic.crenderdata import Include from libclinic.converter import ( CConverter, ConverterType, converters, legacy_converters) from libclinic.converters import ( - self_converter, defining_class_converter, object_converter, buffer, + self_converter, defining_class_converter, buffer, robuffer, rwbuffer, correct_name_for_self) from libclinic.return_converters import ( CReturnConverter, return_converters, int_return_converter, ReturnConverterType) +from libclinic.clanguage import CLanguage +from libclinic.codegen import BlockPrinter, Destination # TODO: @@ -82,1592 +76,6 @@ LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API') -ParamTuple = tuple["Parameter", ...] - - -def permute_left_option_groups( - l: Sequence[Iterable[Parameter]] -) -> Iterator[ParamTuple]: - """ - Given [(1,), (2,), (3,)], should yield: - () - (3,) - (2, 3) - (1, 2, 3) - """ - yield tuple() - accumulator: list[Parameter] = [] - for group in reversed(l): - accumulator = list(group) + accumulator - yield tuple(accumulator) - - -def permute_right_option_groups( - l: Sequence[Iterable[Parameter]] -) -> Iterator[ParamTuple]: - """ - Given [(1,), (2,), (3,)], should yield: - () - (1,) - (1, 2) - (1, 2, 3) - """ - yield tuple() - accumulator: list[Parameter] = [] - for group in l: - accumulator.extend(group) - yield tuple(accumulator) - - -def permute_optional_groups( - left: Sequence[Iterable[Parameter]], - required: Iterable[Parameter], - right: Sequence[Iterable[Parameter]] -) -> tuple[ParamTuple, ...]: - """ - Generator function that computes the set of acceptable - argument lists for the provided iterables of - argument groups. (Actually it generates a tuple of tuples.) - - Algorithm: prefer left options over right options. - - If required is empty, left must also be empty. - """ - required = tuple(required) - if not required: - if left: - raise ValueError("required is empty but left is not") - - accumulator: list[ParamTuple] = [] - counts = set() - for r in permute_right_option_groups(right): - for l in permute_left_option_groups(left): - t = l + required + r - if len(t) in counts: - continue - counts.add(len(t)) - accumulator.append(t) - - accumulator.sort(key=len) - return tuple(accumulator) - - -def declare_parser( - f: Function, - *, - hasformat: bool = False, - clinic: Clinic, - limited_capi: bool, -) -> str: - """ - Generates the code template for a static local PyArg_Parser variable, - with an initializer. For core code (incl. builtin modules) the - kwtuple field is also statically initialized. Otherwise - it is initialized at runtime. - """ - if hasformat: - fname = '' - format_ = '.format = "{format_units}:{name}",' - else: - fname = '.fname = "{name}",' - format_ = '' - - num_keywords = len([ - p for p in f.parameters.values() - if not p.is_positional_only() and not p.is_vararg() - ]) - if limited_capi: - declarations = """ - #define KWTUPLE NULL - """ - elif num_keywords == 0: - declarations = """ - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - """ - else: - declarations = """ - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS %d - static struct {{ - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - }} _kwtuple = {{ - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = {{ {keywords_py} }}, - }}; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - """ % num_keywords - - condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' - clinic.add_include('pycore_gc.h', 'PyGC_Head', condition=condition) - clinic.add_include('pycore_runtime.h', '_Py_ID()', condition=condition) - - declarations += """ - static const char * const _keywords[] = {{{keywords_c} NULL}}; - static _PyArg_Parser _parser = {{ - .keywords = _keywords, - %s - .kwtuple = KWTUPLE, - }}; - #undef KWTUPLE - """ % (format_ or fname) - return libclinic.normalize_snippet(declarations) - - -class CLanguage(Language): - - body_prefix = "#" - language = 'C' - start_line = "/*[{dsl_name} input]" - body_prefix = "" - stop_line = "[{dsl_name} start generated code]*/" - checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" - - NO_VARARG: Final[str] = "PY_SSIZE_T_MAX" - - PARSER_PROTOTYPE_KEYWORD: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) - PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = libclinic.normalize_snippet(""" - static int - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) - PARSER_PROTOTYPE_VARARGS: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args) - """) - PARSER_PROTOTYPE_FASTCALL: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) - """) - PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - PARSER_PROTOTYPE_DEF_CLASS: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - PARSER_PROTOTYPE_NOARGS: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) - """) - PARSER_PROTOTYPE_GETTER: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, void *Py_UNUSED(context)) - """) - PARSER_PROTOTYPE_SETTER: Final[str] = libclinic.normalize_snippet(""" - static int - {c_basename}({self_type}{self_name}, PyObject *value, void *Py_UNUSED(context)) - """) - METH_O_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({impl_parameters}) - """) - DOCSTRING_PROTOTYPE_VAR: Final[str] = libclinic.normalize_snippet(""" - PyDoc_VAR({c_basename}__doc__); - """) - DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" - PyDoc_STRVAR({c_basename}__doc__, - {docstring}); - """) - GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" - PyDoc_STRVAR({getset_basename}__doc__, - {docstring}); - #define {getset_basename}_HAS_DOCSTR - """) - IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" - static {impl_return_type} - {c_basename}_impl({impl_parameters}) - """) - METHODDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #define {methoddef_name} \ - {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, - """) - GETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #if defined({getset_basename}_HAS_DOCSTR) - # define {getset_basename}_DOCSTR {getset_basename}__doc__ - #else - # define {getset_basename}_DOCSTR NULL - #endif - #if defined({getset_name}_GETSETDEF) - # undef {getset_name}_GETSETDEF - # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, - #else - # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, NULL, {getset_basename}_DOCSTR}}, - #endif - """) - SETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #if defined({getset_name}_HAS_DOCSTR) - # define {getset_basename}_DOCSTR {getset_basename}__doc__ - #else - # define {getset_basename}_DOCSTR NULL - #endif - #if defined({getset_name}_GETSETDEF) - # undef {getset_name}_GETSETDEF - # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, - #else - # define {getset_name}_GETSETDEF {{"{name}", NULL, (setter){getset_basename}_set, NULL}}, - #endif - """) - METHODDEF_PROTOTYPE_IFNDEF: Final[str] = libclinic.normalize_snippet(""" - #ifndef {methoddef_name} - #define {methoddef_name} - #endif /* !defined({methoddef_name}) */ - """) - COMPILER_DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" - // Emit compiler warnings when we get to Python {major}.{minor}. - #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 - # error {message} - #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 - # ifdef _MSC_VER - # pragma message ({message}) - # else - # warning {message} - # endif - #endif - """ - DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" - if ({condition}) {{{{{errcheck} - if (PyErr_WarnEx(PyExc_DeprecationWarning, - {message}, 1)) - {{{{ - goto exit; - }}}} - }}}} - """ - - def __init__(self, filename: str) -> None: - super().__init__(filename) - self.cpp = libclinic.cpp.Monitor(filename) - - def parse_line(self, line: str) -> None: - self.cpp.writeline(line) - - def render( - self, - clinic: Clinic, - signatures: Iterable[Module | Class | Function] - ) -> str: - function = None - for o in signatures: - if isinstance(o, Function): - if function: - fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) - function = o - return self.render_function(clinic, function) - - def compiler_deprecated_warning( - self, - func: Function, - parameters: list[Parameter], - ) -> str | None: - minversion: VersionTuple | None = None - for p in parameters: - for version in p.deprecated_positional, p.deprecated_keyword: - if version and (not minversion or minversion > version): - minversion = version - if not minversion: - return None - - # Format the preprocessor warning and error messages. - assert isinstance(self.cpp.filename, str) - message = f"Update the clinic input of {func.full_name!r}." - code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( - major=minversion[0], - minor=minversion[1], - message=libclinic.c_repr(message), - ) - return libclinic.normalize_snippet(code) - - def deprecate_positional_use( - self, - func: Function, - params: dict[int, Parameter], - ) -> str: - assert len(params) > 0 - first_pos = next(iter(params)) - last_pos = next(reversed(params)) - - # Format the deprecation message. - if len(params) == 1: - condition = f"nargs == {first_pos+1}" - amount = f"{first_pos+1} " if first_pos else "" - pl = "s" - else: - condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" - amount = f"more than {first_pos} " if first_pos else "" - pl = "s" if first_pos != 1 else "" - message = ( - f"Passing {amount}positional argument{pl} to " - f"{func.fulldisplayname}() is deprecated." - ) - - for (major, minor), group in itertools.groupby( - params.values(), key=attrgetter("deprecated_positional") - ): - names = [repr(p.name) for p in group] - pstr = libclinic.pprint_words(names) - if len(names) == 1: - message += ( - f" Parameter {pstr} will become a keyword-only parameter " - f"in Python {major}.{minor}." - ) - else: - message += ( - f" Parameters {pstr} will become keyword-only parameters " - f"in Python {major}.{minor}." - ) - - # Append deprecation warning to docstring. - docstring = textwrap.fill(f"Note: {message}") - func.docstring += f"\n\n{docstring}\n" - # Format and return the code block. - code = self.DEPRECATION_WARNING_PROTOTYPE.format( - condition=condition, - errcheck="", - message=libclinic.wrapped_c_string_literal(message, width=64, - subsequent_indent=20), - ) - return libclinic.normalize_snippet(code, indent=4) - - def deprecate_keyword_use( - self, - func: Function, - params: dict[int, Parameter], - argname_fmt: str | None, - *, - fastcall: bool, - limited_capi: bool, - clinic: Clinic, - ) -> str: - assert len(params) > 0 - last_param = next(reversed(params.values())) - - # Format the deprecation message. - containscheck = "" - conditions = [] - for i, p in params.items(): - if p.is_optional(): - if argname_fmt: - conditions.append(f"nargs < {i+1} && {argname_fmt % i}") - elif fastcall: - conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))") - containscheck = "PySequence_Contains" - clinic.add_include('pycore_runtime.h', '_Py_ID()') - else: - conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") - containscheck = "PyDict_Contains" - clinic.add_include('pycore_runtime.h', '_Py_ID()') - else: - conditions = [f"nargs < {i+1}"] - condition = ") || (".join(conditions) - if len(conditions) > 1: - condition = f"(({condition}))" - if last_param.is_optional(): - if fastcall: - if limited_capi: - condition = f"kwnames && PyTuple_Size(kwnames) && {condition}" - else: - condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" - else: - if limited_capi: - condition = f"kwargs && PyDict_Size(kwargs) && {condition}" - else: - condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" - names = [repr(p.name) for p in params.values()] - pstr = libclinic.pprint_words(names) - pl = 's' if len(params) != 1 else '' - message = ( - f"Passing keyword argument{pl} {pstr} to " - f"{func.fulldisplayname}() is deprecated." - ) - - for (major, minor), group in itertools.groupby( - params.values(), key=attrgetter("deprecated_keyword") - ): - names = [repr(p.name) for p in group] - pstr = libclinic.pprint_words(names) - pl = 's' if len(names) != 1 else '' - message += ( - f" Parameter{pl} {pstr} will become positional-only " - f"in Python {major}.{minor}." - ) - - if containscheck: - errcheck = f""" - if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail - goto exit; - }}}}""" - else: - errcheck = "" - if argname_fmt: - # Append deprecation warning to docstring. - docstring = textwrap.fill(f"Note: {message}") - func.docstring += f"\n\n{docstring}\n" - # Format and return the code block. - code = self.DEPRECATION_WARNING_PROTOTYPE.format( - condition=condition, - errcheck=errcheck, - message=libclinic.wrapped_c_string_literal(message, width=64, - subsequent_indent=20), - ) - return libclinic.normalize_snippet(code, indent=4) - - def output_templates( - self, - f: Function, - clinic: Clinic - ) -> dict[str, str]: - parameters = list(f.parameters.values()) - assert parameters - first_param = parameters.pop(0) - assert isinstance(first_param.converter, self_converter) - requires_defining_class = False - if parameters and isinstance(parameters[0].converter, defining_class_converter): - requires_defining_class = True - del parameters[0] - converters = [p.converter for p in parameters] - - if f.critical_section: - clinic.add_include('pycore_critical_section.h', 'Py_BEGIN_CRITICAL_SECTION()') - has_option_groups = parameters and (parameters[0].group or parameters[-1].group) - simple_return = (f.return_converter.type == 'PyObject *' - and not f.critical_section) - new_or_init = f.kind.new_or_init - - vararg: int | str = self.NO_VARARG - pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 - for i, p in enumerate(parameters, 1): - if p.is_keyword_only(): - assert not p.is_positional_only() - if not p.is_optional(): - min_kw_only = i - max_pos - elif p.is_vararg(): - pseudo_args += 1 - vararg = i - 1 - else: - if vararg == self.NO_VARARG: - max_pos = i - if p.is_positional_only(): - pos_only = i - if not p.is_optional(): - min_pos = i - - meth_o = (len(parameters) == 1 and - parameters[0].is_positional_only() and - not converters[0].is_optional() and - not requires_defining_class and - not new_or_init) - - # we have to set these things before we're done: - # - # docstring_prototype - # docstring_definition - # impl_prototype - # methoddef_define - # parser_prototype - # parser_definition - # impl_definition - # cpp_if - # cpp_endif - # methoddef_ifndef - - return_value_declaration = "PyObject *return_value = NULL;" - methoddef_define = self.METHODDEF_PROTOTYPE_DEFINE - if new_or_init and not f.docstring: - docstring_prototype = docstring_definition = '' - elif f.kind is GETTER: - methoddef_define = self.GETTERDEF_PROTOTYPE_DEFINE - if f.docstring: - docstring_prototype = '' - docstring_definition = self.GETSET_DOCSTRING_PROTOTYPE_STRVAR - else: - docstring_prototype = docstring_definition = '' - elif f.kind is SETTER: - if f.docstring: - fail("docstrings are only supported for @getter, not @setter") - return_value_declaration = "int {return_value};" - methoddef_define = self.SETTERDEF_PROTOTYPE_DEFINE - docstring_prototype = docstring_definition = '' - else: - docstring_prototype = self.DOCSTRING_PROTOTYPE_VAR - docstring_definition = self.DOCSTRING_PROTOTYPE_STRVAR - impl_definition = self.IMPL_DEFINITION_PROTOTYPE - impl_prototype = parser_prototype = parser_definition = None - - # parser_body_fields remembers the fields passed in to the - # previous call to parser_body. this is used for an awful hack. - parser_body_fields: tuple[str, ...] = () - def parser_body( - prototype: str, - *fields: str, - declarations: str = '' - ) -> str: - nonlocal parser_body_fields - lines = [] - lines.append(prototype) - parser_body_fields = fields - - preamble = libclinic.normalize_snippet(""" - {{ - {return_value_declaration} - {parser_declarations} - {declarations} - {initializers} - """) + "\n" - finale = libclinic.normalize_snippet(""" - {modifications} - {lock} - {return_value} = {c_basename}_impl({impl_arguments}); - {unlock} - {return_conversion} - {post_parsing} - - {exit_label} - {cleanup} - return return_value; - }} - """) - for field in preamble, *fields, finale: - lines.append(field) - return libclinic.linear_format("\n".join(lines), - parser_declarations=declarations) - - fastcall = not new_or_init - limited_capi = clinic.limited_capi - if limited_capi and (pseudo_args or - (any(p.is_optional() for p in parameters) and - any(p.is_keyword_only() and not p.is_optional() for p in parameters)) or - any(c.broken_limited_capi for c in converters)): - warn(f"Function {f.full_name} cannot use limited C API") - limited_capi = False - - parsearg: str | None - if not parameters: - parser_code: list[str] | None - if f.kind is GETTER: - flags = "" # This should end up unused - parser_prototype = self.PARSER_PROTOTYPE_GETTER - parser_code = [] - elif f.kind is SETTER: - flags = "" - parser_prototype = self.PARSER_PROTOTYPE_SETTER - parser_code = [] - elif not requires_defining_class: - # no parameters, METH_NOARGS - flags = "METH_NOARGS" - parser_prototype = self.PARSER_PROTOTYPE_NOARGS - parser_code = [] - else: - assert fastcall - - flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - return_error = ('return NULL;' if simple_return - else 'goto exit;') - parser_code = [libclinic.normalize_snippet(""" - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) {{ - PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); - %s - }} - """ % return_error, indent=4)] - - if simple_return: - parser_definition = '\n'.join([ - parser_prototype, - '{{', - *parser_code, - ' return {c_basename}_impl({impl_arguments});', - '}}']) - else: - parser_definition = parser_body(parser_prototype, *parser_code) - - elif meth_o: - flags = "METH_O" - - if (isinstance(converters[0], object_converter) and - converters[0].format_unit == 'O'): - meth_o_prototype = self.METH_O_PROTOTYPE - - if simple_return: - # maps perfectly to METH_O, doesn't need a return converter. - # so we skip making a parse function - # and call directly into the impl function. - impl_prototype = parser_prototype = parser_definition = '' - impl_definition = meth_o_prototype - else: - # SLIGHT HACK - # use impl_parameters for the parser here! - parser_prototype = meth_o_prototype - parser_definition = parser_body(parser_prototype) - - else: - argname = 'arg' - if parameters[0].name == argname: - argname += '_' - parser_prototype = libclinic.normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *%s) - """ % argname) - - displayname = parameters[0].get_displayname(0) - parsearg = converters[0].parse_arg(argname, displayname, limited_capi=limited_capi) - if parsearg is None: - converters[0].use_converter() - parsearg = """ - if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ - goto exit; - }} - """ % argname - parser_definition = parser_body(parser_prototype, - libclinic.normalize_snippet(parsearg, indent=4)) - - elif has_option_groups: - # positional parameters with option groups - # (we have to generate lots of PyArg_ParseTuple calls - # in a big switch statement) - - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') - - elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: - if fastcall: - # positional-only, but no option groups - # we only need one call to _PyArg_ParseStack - - flags = "METH_FASTCALL" - parser_prototype = self.PARSER_PROTOTYPE_FASTCALL - nargs = 'nargs' - argname_fmt = 'args[%d]' - else: - # positional-only, but no option groups - # we only need one call to PyArg_ParseTuple - - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - if limited_capi: - nargs = 'PyTuple_Size(args)' - argname_fmt = 'PyTuple_GetItem(args, %d)' - else: - nargs = 'PyTuple_GET_SIZE(args)' - argname_fmt = 'PyTuple_GET_ITEM(args, %d)' - - left_args = f"{nargs} - {max_pos}" - max_args = self.NO_VARARG if (vararg != self.NO_VARARG) else max_pos - if limited_capi: - parser_code = [] - if nargs != 'nargs': - nargs_def = f'Py_ssize_t nargs = {nargs};' - parser_code.append(libclinic.normalize_snippet(nargs_def, indent=4)) - nargs = 'nargs' - if min_pos == max_args: - pl = '' if min_pos == 1 else 's' - parser_code.append(libclinic.normalize_snippet(f""" - if ({nargs} != {min_pos}) {{{{ - PyErr_Format(PyExc_TypeError, "{{name}} expected {min_pos} argument{pl}, got %zd", {nargs}); - goto exit; - }}}} - """, - indent=4)) - else: - if min_pos: - pl = '' if min_pos == 1 else 's' - parser_code.append(libclinic.normalize_snippet(f""" - if ({nargs} < {min_pos}) {{{{ - PyErr_Format(PyExc_TypeError, "{{name}} expected at least {min_pos} argument{pl}, got %zd", {nargs}); - goto exit; - }}}} - """, - indent=4)) - if max_args != self.NO_VARARG: - pl = '' if max_args == 1 else 's' - parser_code.append(libclinic.normalize_snippet(f""" - if ({nargs} > {max_args}) {{{{ - PyErr_Format(PyExc_TypeError, "{{name}} expected at most {max_args} argument{pl}, got %zd", {nargs}); - goto exit; - }}}} - """, - indent=4)) - else: - clinic.add_include('pycore_modsupport.h', - '_PyArg_CheckPositional()') - parser_code = [libclinic.normalize_snippet(f""" - if (!_PyArg_CheckPositional("{{name}}", {nargs}, {min_pos}, {max_args})) {{{{ - goto exit; - }}}} - """, indent=4)] - - has_optional = False - for i, p in enumerate(parameters): - if p.is_vararg(): - if fastcall: - parser_code.append(libclinic.normalize_snippet(""" - %s = PyTuple_New(%s); - if (!%s) {{ - goto exit; - }} - for (Py_ssize_t i = 0; i < %s; ++i) {{ - PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); - }} - """ % ( - p.converter.parser_name, - left_args, - p.converter.parser_name, - left_args, - p.converter.parser_name, - max_pos - ), indent=4)) - else: - parser_code.append(libclinic.normalize_snippet(""" - %s = PyTuple_GetSlice(%d, -1); - """ % ( - p.converter.parser_name, - max_pos - ), indent=4)) - continue - - displayname = p.get_displayname(i+1) - argname = argname_fmt % i - parsearg = p.converter.parse_arg(argname, displayname, limited_capi=limited_capi) - if parsearg is None: - parser_code = None - break - if has_optional or p.is_optional(): - has_optional = True - parser_code.append(libclinic.normalize_snippet(""" - if (%s < %d) {{ - goto skip_optional; - }} - """, indent=4) % (nargs, i + 1)) - parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) - - if parser_code is not None: - if has_optional: - parser_code.append("skip_optional:") - else: - for parameter in parameters: - parameter.converter.use_converter() - - if limited_capi: - fastcall = False - if fastcall: - clinic.add_include('pycore_modsupport.h', - '_PyArg_ParseStack()') - parser_code = [libclinic.normalize_snippet(""" - if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - else: - flags = "METH_VARARGS" - parser_prototype = self.PARSER_PROTOTYPE_VARARGS - parser_code = [libclinic.normalize_snippet(""" - if (!PyArg_ParseTuple(args, "{format_units}:{name}", - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - parser_definition = parser_body(parser_prototype, *parser_code) - - else: - deprecated_positionals: dict[int, Parameter] = {} - deprecated_keywords: dict[int, Parameter] = {} - for i, p in enumerate(parameters): - if p.deprecated_positional: - deprecated_positionals[i] = p - if p.deprecated_keyword: - deprecated_keywords[i] = p - - has_optional_kw = ( - max(pos_only, min_pos) + min_kw_only - < len(converters) - int(vararg != self.NO_VARARG) - ) - - if limited_capi: - parser_code = None - fastcall = False - else: - if vararg == self.NO_VARARG: - clinic.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywords()') - args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only - ) - nargs = "nargs" - else: - clinic.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywordsWithVararg()') - args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only, - vararg - ) - nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" - - if fastcall: - flags = "METH_FASTCALL|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS - argname_fmt = 'args[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [libclinic.normalize_snippet(""" - args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); - if (!args) {{ - goto exit; - }} - """ % args_declaration, indent=4)] - else: - # positional-or-keyword arguments - flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - argname_fmt = 'fastargs[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - declarations += "\nPyObject * const *fastargs;" - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [libclinic.normalize_snippet(""" - fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); - if (!fastargs) {{ - goto exit; - }} - """ % args_declaration, indent=4)] - - if requires_defining_class: - flags = 'METH_METHOD|' + flags - parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - - if parser_code is not None: - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt, - clinic=clinic, - fastcall=fastcall, - limited_capi=limited_capi) - parser_code.append(code) - - add_label: str | None = None - for i, p in enumerate(parameters): - if isinstance(p.converter, defining_class_converter): - raise ValueError("defining_class should be the first " - "parameter (after self)") - displayname = p.get_displayname(i+1) - parsearg = p.converter.parse_arg(argname_fmt % i, displayname, limited_capi=limited_capi) - if parsearg is None: - parser_code = None - break - if add_label and (i == pos_only or i == max_pos): - parser_code.append("%s:" % add_label) - add_label = None - if not p.is_optional(): - parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) - elif i < pos_only: - add_label = 'skip_optional_posonly' - parser_code.append(libclinic.normalize_snippet(""" - if (nargs < %d) {{ - goto %s; - }} - """ % (i + 1, add_label), indent=4)) - if has_optional_kw: - parser_code.append(libclinic.normalize_snippet(""" - noptargs--; - """, indent=4)) - parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) - else: - if i < max_pos: - label = 'skip_optional_pos' - first_opt = max(min_pos, pos_only) - else: - label = 'skip_optional_kwonly' - first_opt = max_pos + min_kw_only - if vararg != self.NO_VARARG: - first_opt += 1 - if i == first_opt: - add_label = label - parser_code.append(libclinic.normalize_snippet(""" - if (!noptargs) {{ - goto %s; - }} - """ % add_label, indent=4)) - if i + 1 == len(parameters): - parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) - else: - add_label = label - parser_code.append(libclinic.normalize_snippet(""" - if (%s) {{ - """ % (argname_fmt % i), indent=4)) - parser_code.append(libclinic.normalize_snippet(parsearg, indent=8)) - parser_code.append(libclinic.normalize_snippet(""" - if (!--noptargs) {{ - goto %s; - }} - }} - """ % add_label, indent=4)) - - if parser_code is not None: - if add_label: - parser_code.append("%s:" % add_label) - else: - for parameter in parameters: - parameter.converter.use_converter() - - declarations = declare_parser(f, clinic=clinic, - hasformat=True, - limited_capi=limited_capi) - if limited_capi: - # positional-or-keyword arguments - assert not fastcall - flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - parser_code = [libclinic.normalize_snippet(""" - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, - {parse_arguments})) - goto exit; - """, indent=4)] - declarations = "static char *_keywords[] = {{{keywords_c} NULL}};" - if deprecated_positionals or deprecated_keywords: - declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" - - elif fastcall: - clinic.add_include('pycore_modsupport.h', - '_PyArg_ParseStackAndKeywords()') - parser_code = [libclinic.normalize_snippet(""" - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - else: - clinic.add_include('pycore_modsupport.h', - '_PyArg_ParseTupleAndKeywordsFast()') - parser_code = [libclinic.normalize_snippet(""" - if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, - {parse_arguments})) {{ - goto exit; - }} - """, indent=4)] - if deprecated_positionals or deprecated_keywords: - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if deprecated_keywords: - code = self.deprecate_keyword_use(f, deprecated_keywords, None, - clinic=clinic, - fastcall=fastcall, - limited_capi=limited_capi) - parser_code.append(code) - - if deprecated_positionals: - code = self.deprecate_positional_use(f, deprecated_positionals) - # Insert the deprecation code before parameter parsing. - parser_code.insert(0, code) - - assert parser_prototype is not None - parser_definition = parser_body(parser_prototype, *parser_code, - declarations=declarations) - - - # Copy includes from parameters to Clinic after parse_arg() has been - # called above. - for converter in converters: - for include in converter.includes: - clinic.add_include(include.filename, include.reason, - condition=include.condition) - - if new_or_init: - methoddef_define = '' - - if f.kind is METHOD_NEW: - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - else: - return_value_declaration = "int return_value = -1;" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD___INIT__ - - fields = list(parser_body_fields) - parses_positional = 'METH_NOARGS' not in flags - parses_keywords = 'METH_KEYWORDS' in flags - if parses_keywords: - assert parses_positional - - if requires_defining_class: - raise ValueError("Slot methods cannot access their defining class.") - - if not parses_keywords: - declarations = '{base_type_ptr}' - clinic.add_include('pycore_modsupport.h', - '_PyArg_NoKeywords()') - fields.insert(0, libclinic.normalize_snippet(""" - if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ - goto exit; - }} - """, indent=4)) - if not parses_positional: - clinic.add_include('pycore_modsupport.h', - '_PyArg_NoPositional()') - fields.insert(0, libclinic.normalize_snippet(""" - if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ - goto exit; - }} - """, indent=4)) - - parser_definition = parser_body(parser_prototype, *fields, - declarations=declarations) - - - methoddef_cast_end = "" - if flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): - methoddef_cast = "(PyCFunction)" - elif f.kind is GETTER: - methoddef_cast = "" # This should end up unused - elif limited_capi: - methoddef_cast = "(PyCFunction)(void(*)(void))" - else: - methoddef_cast = "_PyCFunction_CAST(" - methoddef_cast_end = ")" - - if f.methoddef_flags: - flags += '|' + f.methoddef_flags - - methoddef_define = methoddef_define.replace('{methoddef_flags}', flags) - methoddef_define = methoddef_define.replace('{methoddef_cast}', methoddef_cast) - methoddef_define = methoddef_define.replace('{methoddef_cast_end}', methoddef_cast_end) - - methoddef_ifndef = '' - conditional = self.cpp.condition() - if not conditional: - cpp_if = cpp_endif = '' - else: - cpp_if = "#if " + conditional - cpp_endif = "#endif /* " + conditional + " */" - - if methoddef_define and f.full_name not in clinic.ifndef_symbols: - clinic.ifndef_symbols.add(f.full_name) - methoddef_ifndef = self.METHODDEF_PROTOTYPE_IFNDEF - - # add ';' to the end of parser_prototype and impl_prototype - # (they mustn't be None, but they could be an empty string.) - assert parser_prototype is not None - if parser_prototype: - assert not parser_prototype.endswith(';') - parser_prototype += ';' - - if impl_prototype is None: - impl_prototype = impl_definition - if impl_prototype: - impl_prototype += ";" - - parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration) - - compiler_warning = self.compiler_deprecated_warning(f, parameters) - if compiler_warning: - parser_definition = compiler_warning + "\n\n" + parser_definition - - d = { - "docstring_prototype" : docstring_prototype, - "docstring_definition" : docstring_definition, - "impl_prototype" : impl_prototype, - "methoddef_define" : methoddef_define, - "parser_prototype" : parser_prototype, - "parser_definition" : parser_definition, - "impl_definition" : impl_definition, - "cpp_if" : cpp_if, - "cpp_endif" : cpp_endif, - "methoddef_ifndef" : methoddef_ifndef, - } - - # make sure we didn't forget to assign something, - # and wrap each non-empty value in \n's - d2 = {} - for name, value in d.items(): - assert value is not None, "got a None value for template " + repr(name) - if value: - value = '\n' + value + '\n' - d2[name] = value - return d2 - - @staticmethod - def group_to_variable_name(group: int) -> str: - adjective = "left_" if group < 0 else "right_" - return "group_" + adjective + str(abs(group)) - - def render_option_group_parsing( - self, - f: Function, - template_dict: TemplateDict, - limited_capi: bool, - ) -> None: - # positional only, grouped, optional arguments! - # can be optional on the left or right. - # here's an example: - # - # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] - # - # Here group D are required, and all other groups are optional. - # (Group D's "group" is actually None.) - # We can figure out which sets of arguments we have based on - # how many arguments are in the tuple. - # - # Note that you need to count up on both sides. For example, - # you could have groups C+D, or C+D+E, or C+D+E+F. - # - # What if the number of arguments leads us to an ambiguous result? - # Clinic prefers groups on the left. So in the above example, - # five arguments would map to B+C, not C+D. - - out = [] - parameters = list(f.parameters.values()) - if isinstance(parameters[0].converter, self_converter): - del parameters[0] - - group: list[Parameter] | None = None - left = [] - right = [] - required: list[Parameter] = [] - last: int | Literal[Sentinels.unspecified] = unspecified - - for p in parameters: - group_id = p.group - if group_id != last: - last = group_id - group = [] - if group_id < 0: - left.append(group) - elif group_id == 0: - group = required - else: - right.append(group) - assert group is not None - group.append(p) - - count_min = sys.maxsize - count_max = -1 - - if limited_capi: - nargs = 'PyTuple_Size(args)' - else: - nargs = 'PyTuple_GET_SIZE(args)' - out.append(f"switch ({nargs}) {{\n") - for subset in permute_optional_groups(left, required, right): - count = len(subset) - count_min = min(count_min, count) - count_max = max(count_max, count) - - if count == 0: - out.append(""" case 0: - break; -""") - continue - - group_ids = {p.group for p in subset} # eliminate duplicates - d: dict[str, str | int] = {} - d['count'] = count - d['name'] = f.name - d['format_units'] = "".join(p.converter.format_unit for p in subset) - - parse_arguments: list[str] = [] - for p in subset: - p.converter.parse_argument(parse_arguments) - d['parse_arguments'] = ", ".join(parse_arguments) - - group_ids.discard(0) - lines = "\n".join([ - self.group_to_variable_name(g) + " = 1;" - for g in group_ids - ]) - - s = """\ - case {count}: - if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ - goto exit; - }} - {group_booleans} - break; -""" - s = libclinic.linear_format(s, group_booleans=lines) - s = s.format_map(d) - out.append(s) - - out.append(" default:\n") - s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' - out.append(s.format(f.full_name, count_min, count_max)) - out.append(' goto exit;\n') - out.append("}") - - template_dict['option_group_parsing'] = libclinic.format_escape("".join(out)) - - def render_function( - self, - clinic: Clinic, - f: Function | None - ) -> str: - if f is None or clinic is None: - return "" - - data = CRenderData() - - assert f.parameters, "We should always have a 'self' at this point!" - parameters = f.render_parameters - converters = [p.converter for p in parameters] - - templates = self.output_templates(f, clinic) - - f_self = parameters[0] - selfless = parameters[1:] - assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!" - - if f.critical_section: - match len(f.target_critical_section): - case 0: - lock = 'Py_BEGIN_CRITICAL_SECTION({self_name});' - unlock = 'Py_END_CRITICAL_SECTION();' - case 1: - lock = 'Py_BEGIN_CRITICAL_SECTION({target_critical_section});' - unlock = 'Py_END_CRITICAL_SECTION();' - case _: - lock = 'Py_BEGIN_CRITICAL_SECTION2({target_critical_section});' - unlock = 'Py_END_CRITICAL_SECTION2();' - data.lock.append(lock) - data.unlock.append(unlock) - - last_group = 0 - first_optional = len(selfless) - positional = selfless and selfless[-1].is_positional_only() - has_option_groups = False - - # offset i by -1 because first_optional needs to ignore self - for i, p in enumerate(parameters, -1): - c = p.converter - - if (i != -1) and (p.default is not unspecified): - first_optional = min(first_optional, i) - - if p.is_vararg(): - data.cleanup.append(f"Py_XDECREF({c.parser_name});") - - # insert group variable - group = p.group - if last_group != group: - last_group = group - if group: - group_name = self.group_to_variable_name(group) - data.impl_arguments.append(group_name) - data.declarations.append("int " + group_name + " = 0;") - data.impl_parameters.append("int " + group_name) - has_option_groups = True - - c.render(p, data) - - if has_option_groups and (not positional): - fail("You cannot use optional groups ('[' and ']') " - "unless all parameters are positional-only ('/').") - - # HACK - # when we're METH_O, but have a custom return converter, - # we use "impl_parameters" for the parsing function - # because that works better. but that means we must - # suppress actually declaring the impl's parameters - # as variables in the parsing function. but since it's - # METH_O, we have exactly one anyway, so we know exactly - # where it is. - if ("METH_O" in templates['methoddef_define'] and - '{impl_parameters}' in templates['parser_prototype']): - data.declarations.pop(0) - - full_name = f.full_name - template_dict = {'full_name': full_name} - template_dict['name'] = f.displayname - if f.kind in {GETTER, SETTER}: - template_dict['getset_name'] = f.c_basename.upper() - template_dict['getset_basename'] = f.c_basename - if f.kind is GETTER: - template_dict['c_basename'] = f.c_basename + "_get" - elif f.kind is SETTER: - template_dict['c_basename'] = f.c_basename + "_set" - # Implicitly add the setter value parameter. - data.impl_parameters.append("PyObject *value") - data.impl_arguments.append("value") - else: - template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" - template_dict['c_basename'] = f.c_basename - - template_dict['docstring'] = libclinic.docstring_for_c_string(f.docstring) - template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' - template_dict['target_critical_section'] = ', '.join(f.target_critical_section) - for converter in converters: - converter.set_template_dict(template_dict) - - if f.kind not in {SETTER, METHOD_INIT}: - f.return_converter.render(f, data) - template_dict['impl_return_type'] = f.return_converter.type - - template_dict['declarations'] = libclinic.format_escape("\n".join(data.declarations)) - template_dict['initializers'] = "\n\n".join(data.initializers) - template_dict['modifications'] = '\n\n'.join(data.modifications) - template_dict['keywords_c'] = ' '.join('"' + k + '",' - for k in data.keywords) - keywords = [k for k in data.keywords if k] - template_dict['keywords_py'] = ' '.join('&_Py_ID(' + k + '),' - for k in keywords) - template_dict['format_units'] = ''.join(data.format_units) - template_dict['parse_arguments'] = ', '.join(data.parse_arguments) - if data.parse_arguments: - template_dict['parse_arguments_comma'] = ','; - else: - template_dict['parse_arguments_comma'] = ''; - template_dict['impl_parameters'] = ", ".join(data.impl_parameters) - template_dict['impl_arguments'] = ", ".join(data.impl_arguments) - - template_dict['return_conversion'] = libclinic.format_escape("".join(data.return_conversion).rstrip()) - template_dict['post_parsing'] = libclinic.format_escape("".join(data.post_parsing).rstrip()) - template_dict['cleanup'] = libclinic.format_escape("".join(data.cleanup)) - - template_dict['return_value'] = data.return_value - template_dict['lock'] = "\n".join(data.lock) - template_dict['unlock'] = "\n".join(data.unlock) - - # used by unpack tuple code generator - unpack_min = first_optional - unpack_max = len(selfless) - template_dict['unpack_min'] = str(unpack_min) - template_dict['unpack_max'] = str(unpack_max) - - if has_option_groups: - self.render_option_group_parsing(f, template_dict, - limited_capi=clinic.limited_capi) - - # buffers, not destination - for name, destination in clinic.destination_buffers.items(): - template = templates[name] - if has_option_groups: - template = libclinic.linear_format(template, - option_group_parsing=template_dict['option_group_parsing']) - template = libclinic.linear_format(template, - declarations=template_dict['declarations'], - return_conversion=template_dict['return_conversion'], - initializers=template_dict['initializers'], - modifications=template_dict['modifications'], - post_parsing=template_dict['post_parsing'], - cleanup=template_dict['cleanup'], - lock=template_dict['lock'], - unlock=template_dict['unlock'], - ) - - # Only generate the "exit:" label - # if we have any gotos - label = "exit:" if "goto exit;" in template else "" - template = libclinic.linear_format(template, exit_label=label) - - s = template.format_map(template_dict) - - # mild hack: - # reflow long impl declarations - if name in {"impl_prototype", "impl_definition"}: - s = libclinic.wrap_declarations(s) - - if clinic.line_prefix: - s = libclinic.indent_all_lines(s, clinic.line_prefix) - if clinic.line_suffix: - s = libclinic.suffix_all_lines(s, clinic.line_suffix) - - destination.append(s) - - return clinic.get_destination('block').dump() - - -@dc.dataclass(slots=True) -class BlockPrinter: - language: Language - f: io.StringIO = dc.field(default_factory=io.StringIO) - - # '#include "header.h" // reason': column of '//' comment - INCLUDE_COMMENT_COLUMN: Final[int] = 35 - - def print_block( - self, - block: Block, - *, - core_includes: bool = False, - limited_capi: bool, - header_includes: dict[str, Include], - ) -> None: - input = block.input - output = block.output - dsl_name = block.dsl_name - write = self.f.write - - assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name) - - if not dsl_name: - write(input) - return - - write(self.language.start_line.format(dsl_name=dsl_name)) - write("\n") - - body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) - if not body_prefix: - write(input) - else: - for line in input.split('\n'): - write(body_prefix) - write(line) - write("\n") - - write(self.language.stop_line.format(dsl_name=dsl_name)) - write("\n") - - output = '' - if core_includes and header_includes: - # Emit optional "#include" directives for C headers - output += '\n' - - current_condition: str | None = None - includes = sorted(header_includes.values(), key=Include.sort_key) - for include in includes: - if include.condition != current_condition: - if current_condition: - output += '#endif\n' - current_condition = include.condition - if include.condition: - output += f'{include.condition}\n' - - if current_condition: - line = f'# include "{include.filename}"' - else: - line = f'#include "{include.filename}"' - if include.reason: - comment = f'// {include.reason}\n' - line = line.ljust(self.INCLUDE_COMMENT_COLUMN - 1) + comment - output += line - - if current_condition: - output += '#endif\n' - - input = ''.join(block.input) - output += ''.join(block.output) - if output: - if not output.endswith('\n'): - output += '\n' - write(output) - - arguments = "output={output} input={input}".format( - output=libclinic.compute_checksum(output, 16), - input=libclinic.compute_checksum(input, 16) - ) - write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) - write("\n") - - def write(self, text: str) -> None: - self.f.write(text) - - -class BufferSeries: - """ - Behaves like a "defaultlist". - When you ask for an index that doesn't exist yet, - the object grows the list until that item exists. - So o[n] will always work. - - Supports negative indices for actual items. - e.g. o[-1] is an element immediately preceding o[0]. - """ - - def __init__(self) -> None: - self._start = 0 - self._array: list[list[str]] = [] - - def __getitem__(self, i: int) -> list[str]: - i -= self._start - if i < 0: - self._start += i - prefix: list[list[str]] = [[] for x in range(-i)] - self._array = prefix + self._array - i = 0 - while i >= len(self._array): - self._array.append([]) - return self._array[i] - - def clear(self) -> None: - for ta in self._array: - ta.clear() - - def dump(self) -> str: - texts = ["".join(ta) for ta in self._array] - self.clear() - return "".join(texts) - - -@dc.dataclass(slots=True, repr=False) -class Destination: - name: str - type: str - clinic: Clinic - buffers: BufferSeries = dc.field(init=False, default_factory=BufferSeries) - filename: str = dc.field(init=False) # set in __post_init__ - - args: dc.InitVar[tuple[str, ...]] = () - - def __post_init__(self, args: tuple[str, ...]) -> None: - valid_types = ('buffer', 'file', 'suppress') - if self.type not in valid_types: - fail( - f"Invalid destination type {self.type!r} for {self.name}, " - f"must be {', '.join(valid_types)}" - ) - extra_arguments = 1 if self.type == "file" else 0 - if len(args) < extra_arguments: - fail(f"Not enough arguments for destination " - f"{self.name!r} new {self.type!r}") - if len(args) > extra_arguments: - fail(f"Too many arguments for destination {self.name!r} new {self.type!r}") - if self.type =='file': - d = {} - filename = self.clinic.filename - d['path'] = filename - dirname, basename = os.path.split(filename) - if not dirname: - dirname = '.' - d['dirname'] = dirname - d['basename'] = basename - d['basename_root'], d['basename_extension'] = os.path.splitext(filename) - self.filename = args[0].format_map(d) - - def __repr__(self) -> str: - if self.type == 'file': - type_repr = f"type='file' file={self.filename!r}" - else: - type_repr = f"type={self.type!r}" - return f"" - - def clear(self) -> None: - if self.type != 'buffer': - fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'") - self.buffers.clear() - - def dump(self) -> str: - return self.buffers.dump() - - # "extensions" maps the file extension ("c", "py") to Language classes. LangDict = dict[str, Callable[[str], Language]] extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py new file mode 100644 index 00000000000000..3f4ca4aab56d67 --- /dev/null +++ b/Tools/clinic/libclinic/clanguage.py @@ -0,0 +1,1364 @@ +from __future__ import annotations +import itertools +import sys +import textwrap +from typing import TYPE_CHECKING, Literal, Final +from operator import attrgetter +from collections.abc import Iterable + +import libclinic +from libclinic import ( + unspecified, fail, warn, Sentinels, VersionTuple) +from libclinic.function import ( + GETTER, SETTER, METHOD_INIT, METHOD_NEW) +from libclinic.crenderdata import CRenderData, TemplateDict +from libclinic.language import Language +from libclinic.function import ( + Module, Class, Function, Parameter, + permute_optional_groups) +from libclinic.converters import ( + defining_class_converter, object_converter, self_converter) +if TYPE_CHECKING: + from clinic import Clinic + + +def declare_parser( + f: Function, + *, + hasformat: bool = False, + clinic: Clinic, + limited_capi: bool, +) -> str: + """ + Generates the code template for a static local PyArg_Parser variable, + with an initializer. For core code (incl. builtin modules) the + kwtuple field is also statically initialized. Otherwise + it is initialized at runtime. + """ + if hasformat: + fname = '' + format_ = '.format = "{format_units}:{name}",' + else: + fname = '.fname = "{name}",' + format_ = '' + + num_keywords = len([ + p for p in f.parameters.values() + if not p.is_positional_only() and not p.is_vararg() + ]) + if limited_capi: + declarations = """ + #define KWTUPLE NULL + """ + elif num_keywords == 0: + declarations = """ + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + """ + else: + declarations = """ + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS %d + static struct {{ + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + }} _kwtuple = {{ + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = {{ {keywords_py} }}, + }}; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + """ % num_keywords + + condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' + clinic.add_include('pycore_gc.h', 'PyGC_Head', condition=condition) + clinic.add_include('pycore_runtime.h', '_Py_ID()', condition=condition) + + declarations += """ + static const char * const _keywords[] = {{{keywords_c} NULL}}; + static _PyArg_Parser _parser = {{ + .keywords = _keywords, + %s + .kwtuple = KWTUPLE, + }}; + #undef KWTUPLE + """ % (format_ or fname) + return libclinic.normalize_snippet(declarations) + + +class CLanguage(Language): + + body_prefix = "#" + language = 'C' + start_line = "/*[{dsl_name} input]" + body_prefix = "" + stop_line = "[{dsl_name} start generated code]*/" + checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" + + NO_VARARG: Final[str] = "PY_SSIZE_T_MAX" + + PARSER_PROTOTYPE_KEYWORD: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) + """) + PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = libclinic.normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) + """) + PARSER_PROTOTYPE_VARARGS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args) + """) + PARSER_PROTOTYPE_FASTCALL: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) + """) + PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + """) + PARSER_PROTOTYPE_DEF_CLASS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + """) + PARSER_PROTOTYPE_NOARGS: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) + """) + PARSER_PROTOTYPE_GETTER: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, void *Py_UNUSED(context)) + """) + PARSER_PROTOTYPE_SETTER: Final[str] = libclinic.normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *value, void *Py_UNUSED(context)) + """) + METH_O_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({impl_parameters}) + """) + DOCSTRING_PROTOTYPE_VAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_VAR({c_basename}__doc__); + """) + DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_STRVAR({c_basename}__doc__, + {docstring}); + """) + GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" + PyDoc_STRVAR({getset_basename}__doc__, + {docstring}); + #define {getset_basename}_HAS_DOCSTR + """) + IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" + static {impl_return_type} + {c_basename}_impl({impl_parameters}) + """) + METHODDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #define {methoddef_name} \ + {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, + """) + GETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #if defined({getset_basename}_HAS_DOCSTR) + # define {getset_basename}_DOCSTR {getset_basename}__doc__ + #else + # define {getset_basename}_DOCSTR NULL + #endif + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, NULL, {getset_basename}_DOCSTR}}, + #endif + """) + SETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" + #if defined({getset_name}_HAS_DOCSTR) + # define {getset_basename}_DOCSTR {getset_basename}__doc__ + #else + # define {getset_basename}_DOCSTR NULL + #endif + #if defined({getset_name}_GETSETDEF) + # undef {getset_name}_GETSETDEF + # define {getset_name}_GETSETDEF {{"{name}", (getter){getset_basename}_get, (setter){getset_basename}_set, {getset_basename}_DOCSTR}}, + #else + # define {getset_name}_GETSETDEF {{"{name}", NULL, (setter){getset_basename}_set, NULL}}, + #endif + """) + METHODDEF_PROTOTYPE_IFNDEF: Final[str] = libclinic.normalize_snippet(""" + #ifndef {methoddef_name} + #define {methoddef_name} + #endif /* !defined({methoddef_name}) */ + """) + COMPILER_DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" + // Emit compiler warnings when we get to Python {major}.{minor}. + #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 + # error {message} + #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 + # ifdef _MSC_VER + # pragma message ({message}) + # else + # warning {message} + # endif + #endif + """ + DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" + if ({condition}) {{{{{errcheck} + if (PyErr_WarnEx(PyExc_DeprecationWarning, + {message}, 1)) + {{{{ + goto exit; + }}}} + }}}} + """ + + def __init__(self, filename: str) -> None: + super().__init__(filename) + self.cpp = libclinic.cpp.Monitor(filename) + + def parse_line(self, line: str) -> None: + self.cpp.writeline(line) + + def render( + self, + clinic: Clinic, + signatures: Iterable[Module | Class | Function] + ) -> str: + function = None + for o in signatures: + if isinstance(o, Function): + if function: + fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) + function = o + return self.render_function(clinic, function) + + def compiler_deprecated_warning( + self, + func: Function, + parameters: list[Parameter], + ) -> str | None: + minversion: VersionTuple | None = None + for p in parameters: + for version in p.deprecated_positional, p.deprecated_keyword: + if version and (not minversion or minversion > version): + minversion = version + if not minversion: + return None + + # Format the preprocessor warning and error messages. + assert isinstance(self.cpp.filename, str) + message = f"Update the clinic input of {func.full_name!r}." + code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( + major=minversion[0], + minor=minversion[1], + message=libclinic.c_repr(message), + ) + return libclinic.normalize_snippet(code) + + def deprecate_positional_use( + self, + func: Function, + params: dict[int, Parameter], + ) -> str: + assert len(params) > 0 + first_pos = next(iter(params)) + last_pos = next(reversed(params)) + + # Format the deprecation message. + if len(params) == 1: + condition = f"nargs == {first_pos+1}" + amount = f"{first_pos+1} " if first_pos else "" + pl = "s" + else: + condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" + amount = f"more than {first_pos} " if first_pos else "" + pl = "s" if first_pos != 1 else "" + message = ( + f"Passing {amount}positional argument{pl} to " + f"{func.fulldisplayname}() is deprecated." + ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_positional") + ): + names = [repr(p.name) for p in group] + pstr = libclinic.pprint_words(names) + if len(names) == 1: + message += ( + f" Parameter {pstr} will become a keyword-only parameter " + f"in Python {major}.{minor}." + ) + else: + message += ( + f" Parameters {pstr} will become keyword-only parameters " + f"in Python {major}.{minor}." + ) + + # Append deprecation warning to docstring. + docstring = textwrap.fill(f"Note: {message}") + func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. + code = self.DEPRECATION_WARNING_PROTOTYPE.format( + condition=condition, + errcheck="", + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), + ) + return libclinic.normalize_snippet(code, indent=4) + + def deprecate_keyword_use( + self, + func: Function, + params: dict[int, Parameter], + argname_fmt: str | None, + *, + fastcall: bool, + limited_capi: bool, + clinic: Clinic, + ) -> str: + assert len(params) > 0 + last_param = next(reversed(params.values())) + + # Format the deprecation message. + containscheck = "" + conditions = [] + for i, p in params.items(): + if p.is_optional(): + if argname_fmt: + conditions.append(f"nargs < {i+1} && {argname_fmt % i}") + elif fastcall: + conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))") + containscheck = "PySequence_Contains" + clinic.add_include('pycore_runtime.h', '_Py_ID()') + else: + conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") + containscheck = "PyDict_Contains" + clinic.add_include('pycore_runtime.h', '_Py_ID()') + else: + conditions = [f"nargs < {i+1}"] + condition = ") || (".join(conditions) + if len(conditions) > 1: + condition = f"(({condition}))" + if last_param.is_optional(): + if fastcall: + if limited_capi: + condition = f"kwnames && PyTuple_Size(kwnames) && {condition}" + else: + condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" + else: + if limited_capi: + condition = f"kwargs && PyDict_Size(kwargs) && {condition}" + else: + condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" + names = [repr(p.name) for p in params.values()] + pstr = libclinic.pprint_words(names) + pl = 's' if len(params) != 1 else '' + message = ( + f"Passing keyword argument{pl} {pstr} to " + f"{func.fulldisplayname}() is deprecated." + ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_keyword") + ): + names = [repr(p.name) for p in group] + pstr = libclinic.pprint_words(names) + pl = 's' if len(names) != 1 else '' + message += ( + f" Parameter{pl} {pstr} will become positional-only " + f"in Python {major}.{minor}." + ) + + if containscheck: + errcheck = f""" + if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail + goto exit; + }}}}""" + else: + errcheck = "" + if argname_fmt: + # Append deprecation warning to docstring. + docstring = textwrap.fill(f"Note: {message}") + func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. + code = self.DEPRECATION_WARNING_PROTOTYPE.format( + condition=condition, + errcheck=errcheck, + message=libclinic.wrapped_c_string_literal(message, width=64, + subsequent_indent=20), + ) + return libclinic.normalize_snippet(code, indent=4) + + def output_templates( + self, + f: Function, + clinic: Clinic + ) -> dict[str, str]: + parameters = list(f.parameters.values()) + assert parameters + first_param = parameters.pop(0) + assert isinstance(first_param.converter, self_converter) + requires_defining_class = False + if parameters and isinstance(parameters[0].converter, defining_class_converter): + requires_defining_class = True + del parameters[0] + converters = [p.converter for p in parameters] + + if f.critical_section: + clinic.add_include('pycore_critical_section.h', 'Py_BEGIN_CRITICAL_SECTION()') + has_option_groups = parameters and (parameters[0].group or parameters[-1].group) + simple_return = (f.return_converter.type == 'PyObject *' + and not f.critical_section) + new_or_init = f.kind.new_or_init + + vararg: int | str = self.NO_VARARG + pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 + for i, p in enumerate(parameters, 1): + if p.is_keyword_only(): + assert not p.is_positional_only() + if not p.is_optional(): + min_kw_only = i - max_pos + elif p.is_vararg(): + pseudo_args += 1 + vararg = i - 1 + else: + if vararg == self.NO_VARARG: + max_pos = i + if p.is_positional_only(): + pos_only = i + if not p.is_optional(): + min_pos = i + + meth_o = (len(parameters) == 1 and + parameters[0].is_positional_only() and + not converters[0].is_optional() and + not requires_defining_class and + not new_or_init) + + # we have to set these things before we're done: + # + # docstring_prototype + # docstring_definition + # impl_prototype + # methoddef_define + # parser_prototype + # parser_definition + # impl_definition + # cpp_if + # cpp_endif + # methoddef_ifndef + + return_value_declaration = "PyObject *return_value = NULL;" + methoddef_define = self.METHODDEF_PROTOTYPE_DEFINE + if new_or_init and not f.docstring: + docstring_prototype = docstring_definition = '' + elif f.kind is GETTER: + methoddef_define = self.GETTERDEF_PROTOTYPE_DEFINE + if f.docstring: + docstring_prototype = '' + docstring_definition = self.GETSET_DOCSTRING_PROTOTYPE_STRVAR + else: + docstring_prototype = docstring_definition = '' + elif f.kind is SETTER: + if f.docstring: + fail("docstrings are only supported for @getter, not @setter") + return_value_declaration = "int {return_value};" + methoddef_define = self.SETTERDEF_PROTOTYPE_DEFINE + docstring_prototype = docstring_definition = '' + else: + docstring_prototype = self.DOCSTRING_PROTOTYPE_VAR + docstring_definition = self.DOCSTRING_PROTOTYPE_STRVAR + impl_definition = self.IMPL_DEFINITION_PROTOTYPE + impl_prototype = parser_prototype = parser_definition = None + + # parser_body_fields remembers the fields passed in to the + # previous call to parser_body. this is used for an awful hack. + parser_body_fields: tuple[str, ...] = () + def parser_body( + prototype: str, + *fields: str, + declarations: str = '' + ) -> str: + nonlocal parser_body_fields + lines = [] + lines.append(prototype) + parser_body_fields = fields + + preamble = libclinic.normalize_snippet(""" + {{ + {return_value_declaration} + {parser_declarations} + {declarations} + {initializers} + """) + "\n" + finale = libclinic.normalize_snippet(""" + {modifications} + {lock} + {return_value} = {c_basename}_impl({impl_arguments}); + {unlock} + {return_conversion} + {post_parsing} + + {exit_label} + {cleanup} + return return_value; + }} + """) + for field in preamble, *fields, finale: + lines.append(field) + return libclinic.linear_format("\n".join(lines), + parser_declarations=declarations) + + fastcall = not new_or_init + limited_capi = clinic.limited_capi + if limited_capi and (pseudo_args or + (any(p.is_optional() for p in parameters) and + any(p.is_keyword_only() and not p.is_optional() for p in parameters)) or + any(c.broken_limited_capi for c in converters)): + warn(f"Function {f.full_name} cannot use limited C API") + limited_capi = False + + parsearg: str | None + if not parameters: + parser_code: list[str] | None + if f.kind is GETTER: + flags = "" # This should end up unused + parser_prototype = self.PARSER_PROTOTYPE_GETTER + parser_code = [] + elif f.kind is SETTER: + flags = "" + parser_prototype = self.PARSER_PROTOTYPE_SETTER + parser_code = [] + elif not requires_defining_class: + # no parameters, METH_NOARGS + flags = "METH_NOARGS" + parser_prototype = self.PARSER_PROTOTYPE_NOARGS + parser_code = [] + else: + assert fastcall + + flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" + parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS + return_error = ('return NULL;' if simple_return + else 'goto exit;') + parser_code = [libclinic.normalize_snippet(""" + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) {{ + PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); + %s + }} + """ % return_error, indent=4)] + + if simple_return: + parser_definition = '\n'.join([ + parser_prototype, + '{{', + *parser_code, + ' return {c_basename}_impl({impl_arguments});', + '}}']) + else: + parser_definition = parser_body(parser_prototype, *parser_code) + + elif meth_o: + flags = "METH_O" + + if (isinstance(converters[0], object_converter) and + converters[0].format_unit == 'O'): + meth_o_prototype = self.METH_O_PROTOTYPE + + if simple_return: + # maps perfectly to METH_O, doesn't need a return converter. + # so we skip making a parse function + # and call directly into the impl function. + impl_prototype = parser_prototype = parser_definition = '' + impl_definition = meth_o_prototype + else: + # SLIGHT HACK + # use impl_parameters for the parser here! + parser_prototype = meth_o_prototype + parser_definition = parser_body(parser_prototype) + + else: + argname = 'arg' + if parameters[0].name == argname: + argname += '_' + parser_prototype = libclinic.normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *%s) + """ % argname) + + displayname = parameters[0].get_displayname(0) + parsearg = converters[0].parse_arg(argname, displayname, limited_capi=limited_capi) + if parsearg is None: + converters[0].use_converter() + parsearg = """ + if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ + goto exit; + }} + """ % argname + parser_definition = parser_body(parser_prototype, + libclinic.normalize_snippet(parsearg, indent=4)) + + elif has_option_groups: + # positional parameters with option groups + # (we have to generate lots of PyArg_ParseTuple calls + # in a big switch statement) + + flags = "METH_VARARGS" + parser_prototype = self.PARSER_PROTOTYPE_VARARGS + parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') + + elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: + if fastcall: + # positional-only, but no option groups + # we only need one call to _PyArg_ParseStack + + flags = "METH_FASTCALL" + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL + nargs = 'nargs' + argname_fmt = 'args[%d]' + else: + # positional-only, but no option groups + # we only need one call to PyArg_ParseTuple + + flags = "METH_VARARGS" + parser_prototype = self.PARSER_PROTOTYPE_VARARGS + if limited_capi: + nargs = 'PyTuple_Size(args)' + argname_fmt = 'PyTuple_GetItem(args, %d)' + else: + nargs = 'PyTuple_GET_SIZE(args)' + argname_fmt = 'PyTuple_GET_ITEM(args, %d)' + + left_args = f"{nargs} - {max_pos}" + max_args = self.NO_VARARG if (vararg != self.NO_VARARG) else max_pos + if limited_capi: + parser_code = [] + if nargs != 'nargs': + nargs_def = f'Py_ssize_t nargs = {nargs};' + parser_code.append(libclinic.normalize_snippet(nargs_def, indent=4)) + nargs = 'nargs' + if min_pos == max_args: + pl = '' if min_pos == 1 else 's' + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} != {min_pos}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected {min_pos} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + else: + if min_pos: + pl = '' if min_pos == 1 else 's' + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} < {min_pos}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected at least {min_pos} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + if max_args != self.NO_VARARG: + pl = '' if max_args == 1 else 's' + parser_code.append(libclinic.normalize_snippet(f""" + if ({nargs} > {max_args}) {{{{ + PyErr_Format(PyExc_TypeError, "{{name}} expected at most {max_args} argument{pl}, got %zd", {nargs}); + goto exit; + }}}} + """, + indent=4)) + else: + clinic.add_include('pycore_modsupport.h', + '_PyArg_CheckPositional()') + parser_code = [libclinic.normalize_snippet(f""" + if (!_PyArg_CheckPositional("{{name}}", {nargs}, {min_pos}, {max_args})) {{{{ + goto exit; + }}}} + """, indent=4)] + + has_optional = False + for i, p in enumerate(parameters): + if p.is_vararg(): + if fastcall: + parser_code.append(libclinic.normalize_snippet(""" + %s = PyTuple_New(%s); + if (!%s) {{ + goto exit; + }} + for (Py_ssize_t i = 0; i < %s; ++i) {{ + PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); + }} + """ % ( + p.converter.parser_name, + left_args, + p.converter.parser_name, + left_args, + p.converter.parser_name, + max_pos + ), indent=4)) + else: + parser_code.append(libclinic.normalize_snippet(""" + %s = PyTuple_GetSlice(%d, -1); + """ % ( + p.converter.parser_name, + max_pos + ), indent=4)) + continue + + displayname = p.get_displayname(i+1) + argname = argname_fmt % i + parsearg = p.converter.parse_arg(argname, displayname, limited_capi=limited_capi) + if parsearg is None: + parser_code = None + break + if has_optional or p.is_optional(): + has_optional = True + parser_code.append(libclinic.normalize_snippet(""" + if (%s < %d) {{ + goto skip_optional; + }} + """, indent=4) % (nargs, i + 1)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + + if parser_code is not None: + if has_optional: + parser_code.append("skip_optional:") + else: + for parameter in parameters: + parameter.converter.use_converter() + + if limited_capi: + fastcall = False + if fastcall: + clinic.add_include('pycore_modsupport.h', + '_PyArg_ParseStack()') + parser_code = [libclinic.normalize_snippet(""" + if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + else: + flags = "METH_VARARGS" + parser_prototype = self.PARSER_PROTOTYPE_VARARGS + parser_code = [libclinic.normalize_snippet(""" + if (!PyArg_ParseTuple(args, "{format_units}:{name}", + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + parser_definition = parser_body(parser_prototype, *parser_code) + + else: + deprecated_positionals: dict[int, Parameter] = {} + deprecated_keywords: dict[int, Parameter] = {} + for i, p in enumerate(parameters): + if p.deprecated_positional: + deprecated_positionals[i] = p + if p.deprecated_keyword: + deprecated_keywords[i] = p + + has_optional_kw = ( + max(pos_only, min_pos) + min_kw_only + < len(converters) - int(vararg != self.NO_VARARG) + ) + + if limited_capi: + parser_code = None + fastcall = False + else: + if vararg == self.NO_VARARG: + clinic.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywords()') + args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( + min_pos, + max_pos, + min_kw_only + ) + nargs = "nargs" + else: + clinic.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywordsWithVararg()') + args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( + min_pos, + max_pos, + min_kw_only, + vararg + ) + nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" + + if fastcall: + flags = "METH_FASTCALL|METH_KEYWORDS" + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS + argname_fmt = 'args[%d]' + declarations = declare_parser(f, clinic=clinic, + limited_capi=clinic.limited_capi) + declarations += "\nPyObject *argsbuf[%s];" % len(converters) + if has_optional_kw: + declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) + parser_code = [libclinic.normalize_snippet(""" + args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); + if (!args) {{ + goto exit; + }} + """ % args_declaration, indent=4)] + else: + # positional-or-keyword arguments + flags = "METH_VARARGS|METH_KEYWORDS" + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD + argname_fmt = 'fastargs[%d]' + declarations = declare_parser(f, clinic=clinic, + limited_capi=clinic.limited_capi) + declarations += "\nPyObject *argsbuf[%s];" % len(converters) + declarations += "\nPyObject * const *fastargs;" + declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if has_optional_kw: + declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) + parser_code = [libclinic.normalize_snippet(""" + fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); + if (!fastargs) {{ + goto exit; + }} + """ % args_declaration, indent=4)] + + if requires_defining_class: + flags = 'METH_METHOD|' + flags + parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS + + if parser_code is not None: + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt, + clinic=clinic, + fastcall=fastcall, + limited_capi=limited_capi) + parser_code.append(code) + + add_label: str | None = None + for i, p in enumerate(parameters): + if isinstance(p.converter, defining_class_converter): + raise ValueError("defining_class should be the first " + "parameter (after self)") + displayname = p.get_displayname(i+1) + parsearg = p.converter.parse_arg(argname_fmt % i, displayname, limited_capi=limited_capi) + if parsearg is None: + parser_code = None + break + if add_label and (i == pos_only or i == max_pos): + parser_code.append("%s:" % add_label) + add_label = None + if not p.is_optional(): + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + elif i < pos_only: + add_label = 'skip_optional_posonly' + parser_code.append(libclinic.normalize_snippet(""" + if (nargs < %d) {{ + goto %s; + }} + """ % (i + 1, add_label), indent=4)) + if has_optional_kw: + parser_code.append(libclinic.normalize_snippet(""" + noptargs--; + """, indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + else: + if i < max_pos: + label = 'skip_optional_pos' + first_opt = max(min_pos, pos_only) + else: + label = 'skip_optional_kwonly' + first_opt = max_pos + min_kw_only + if vararg != self.NO_VARARG: + first_opt += 1 + if i == first_opt: + add_label = label + parser_code.append(libclinic.normalize_snippet(""" + if (!noptargs) {{ + goto %s; + }} + """ % add_label, indent=4)) + if i + 1 == len(parameters): + parser_code.append(libclinic.normalize_snippet(parsearg, indent=4)) + else: + add_label = label + parser_code.append(libclinic.normalize_snippet(""" + if (%s) {{ + """ % (argname_fmt % i), indent=4)) + parser_code.append(libclinic.normalize_snippet(parsearg, indent=8)) + parser_code.append(libclinic.normalize_snippet(""" + if (!--noptargs) {{ + goto %s; + }} + }} + """ % add_label, indent=4)) + + if parser_code is not None: + if add_label: + parser_code.append("%s:" % add_label) + else: + for parameter in parameters: + parameter.converter.use_converter() + + declarations = declare_parser(f, clinic=clinic, + hasformat=True, + limited_capi=limited_capi) + if limited_capi: + # positional-or-keyword arguments + assert not fastcall + flags = "METH_VARARGS|METH_KEYWORDS" + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD + parser_code = [libclinic.normalize_snippet(""" + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{format_units}:{name}", _keywords, + {parse_arguments})) + goto exit; + """, indent=4)] + declarations = "static char *_keywords[] = {{{keywords_c} NULL}};" + if deprecated_positionals or deprecated_keywords: + declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);" + + elif fastcall: + clinic.add_include('pycore_modsupport.h', + '_PyArg_ParseStackAndKeywords()') + parser_code = [libclinic.normalize_snippet(""" + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + else: + clinic.add_include('pycore_modsupport.h', + '_PyArg_ParseTupleAndKeywordsFast()') + parser_code = [libclinic.normalize_snippet(""" + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + {parse_arguments})) {{ + goto exit; + }} + """, indent=4)] + if deprecated_positionals or deprecated_keywords: + declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, None, + clinic=clinic, + fastcall=fastcall, + limited_capi=limited_capi) + parser_code.append(code) + + if deprecated_positionals: + code = self.deprecate_positional_use(f, deprecated_positionals) + # Insert the deprecation code before parameter parsing. + parser_code.insert(0, code) + + assert parser_prototype is not None + parser_definition = parser_body(parser_prototype, *parser_code, + declarations=declarations) + + + # Copy includes from parameters to Clinic after parse_arg() has been + # called above. + for converter in converters: + for include in converter.includes: + clinic.add_include(include.filename, include.reason, + condition=include.condition) + + if new_or_init: + methoddef_define = '' + + if f.kind is METHOD_NEW: + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD + else: + return_value_declaration = "int return_value = -1;" + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD___INIT__ + + fields = list(parser_body_fields) + parses_positional = 'METH_NOARGS' not in flags + parses_keywords = 'METH_KEYWORDS' in flags + if parses_keywords: + assert parses_positional + + if requires_defining_class: + raise ValueError("Slot methods cannot access their defining class.") + + if not parses_keywords: + declarations = '{base_type_ptr}' + clinic.add_include('pycore_modsupport.h', + '_PyArg_NoKeywords()') + fields.insert(0, libclinic.normalize_snippet(""" + if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ + goto exit; + }} + """, indent=4)) + if not parses_positional: + clinic.add_include('pycore_modsupport.h', + '_PyArg_NoPositional()') + fields.insert(0, libclinic.normalize_snippet(""" + if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ + goto exit; + }} + """, indent=4)) + + parser_definition = parser_body(parser_prototype, *fields, + declarations=declarations) + + + methoddef_cast_end = "" + if flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): + methoddef_cast = "(PyCFunction)" + elif f.kind is GETTER: + methoddef_cast = "" # This should end up unused + elif limited_capi: + methoddef_cast = "(PyCFunction)(void(*)(void))" + else: + methoddef_cast = "_PyCFunction_CAST(" + methoddef_cast_end = ")" + + if f.methoddef_flags: + flags += '|' + f.methoddef_flags + + methoddef_define = methoddef_define.replace('{methoddef_flags}', flags) + methoddef_define = methoddef_define.replace('{methoddef_cast}', methoddef_cast) + methoddef_define = methoddef_define.replace('{methoddef_cast_end}', methoddef_cast_end) + + methoddef_ifndef = '' + conditional = self.cpp.condition() + if not conditional: + cpp_if = cpp_endif = '' + else: + cpp_if = "#if " + conditional + cpp_endif = "#endif /* " + conditional + " */" + + if methoddef_define and f.full_name not in clinic.ifndef_symbols: + clinic.ifndef_symbols.add(f.full_name) + methoddef_ifndef = self.METHODDEF_PROTOTYPE_IFNDEF + + # add ';' to the end of parser_prototype and impl_prototype + # (they mustn't be None, but they could be an empty string.) + assert parser_prototype is not None + if parser_prototype: + assert not parser_prototype.endswith(';') + parser_prototype += ';' + + if impl_prototype is None: + impl_prototype = impl_definition + if impl_prototype: + impl_prototype += ";" + + parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration) + + compiler_warning = self.compiler_deprecated_warning(f, parameters) + if compiler_warning: + parser_definition = compiler_warning + "\n\n" + parser_definition + + d = { + "docstring_prototype" : docstring_prototype, + "docstring_definition" : docstring_definition, + "impl_prototype" : impl_prototype, + "methoddef_define" : methoddef_define, + "parser_prototype" : parser_prototype, + "parser_definition" : parser_definition, + "impl_definition" : impl_definition, + "cpp_if" : cpp_if, + "cpp_endif" : cpp_endif, + "methoddef_ifndef" : methoddef_ifndef, + } + + # make sure we didn't forget to assign something, + # and wrap each non-empty value in \n's + d2 = {} + for name, value in d.items(): + assert value is not None, "got a None value for template " + repr(name) + if value: + value = '\n' + value + '\n' + d2[name] = value + return d2 + + @staticmethod + def group_to_variable_name(group: int) -> str: + adjective = "left_" if group < 0 else "right_" + return "group_" + adjective + str(abs(group)) + + def render_option_group_parsing( + self, + f: Function, + template_dict: TemplateDict, + limited_capi: bool, + ) -> None: + # positional only, grouped, optional arguments! + # can be optional on the left or right. + # here's an example: + # + # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] + # + # Here group D are required, and all other groups are optional. + # (Group D's "group" is actually None.) + # We can figure out which sets of arguments we have based on + # how many arguments are in the tuple. + # + # Note that you need to count up on both sides. For example, + # you could have groups C+D, or C+D+E, or C+D+E+F. + # + # What if the number of arguments leads us to an ambiguous result? + # Clinic prefers groups on the left. So in the above example, + # five arguments would map to B+C, not C+D. + + out = [] + parameters = list(f.parameters.values()) + if isinstance(parameters[0].converter, self_converter): + del parameters[0] + + group: list[Parameter] | None = None + left = [] + right = [] + required: list[Parameter] = [] + last: int | Literal[Sentinels.unspecified] = unspecified + + for p in parameters: + group_id = p.group + if group_id != last: + last = group_id + group = [] + if group_id < 0: + left.append(group) + elif group_id == 0: + group = required + else: + right.append(group) + assert group is not None + group.append(p) + + count_min = sys.maxsize + count_max = -1 + + if limited_capi: + nargs = 'PyTuple_Size(args)' + else: + nargs = 'PyTuple_GET_SIZE(args)' + out.append(f"switch ({nargs}) {{\n") + for subset in permute_optional_groups(left, required, right): + count = len(subset) + count_min = min(count_min, count) + count_max = max(count_max, count) + + if count == 0: + out.append(""" case 0: + break; +""") + continue + + group_ids = {p.group for p in subset} # eliminate duplicates + d: dict[str, str | int] = {} + d['count'] = count + d['name'] = f.name + d['format_units'] = "".join(p.converter.format_unit for p in subset) + + parse_arguments: list[str] = [] + for p in subset: + p.converter.parse_argument(parse_arguments) + d['parse_arguments'] = ", ".join(parse_arguments) + + group_ids.discard(0) + lines = "\n".join([ + self.group_to_variable_name(g) + " = 1;" + for g in group_ids + ]) + + s = """\ + case {count}: + if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ + goto exit; + }} + {group_booleans} + break; +""" + s = libclinic.linear_format(s, group_booleans=lines) + s = s.format_map(d) + out.append(s) + + out.append(" default:\n") + s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' + out.append(s.format(f.full_name, count_min, count_max)) + out.append(' goto exit;\n') + out.append("}") + + template_dict['option_group_parsing'] = libclinic.format_escape("".join(out)) + + def render_function( + self, + clinic: Clinic, + f: Function | None + ) -> str: + if f is None or clinic is None: + return "" + + data = CRenderData() + + assert f.parameters, "We should always have a 'self' at this point!" + parameters = f.render_parameters + converters = [p.converter for p in parameters] + + templates = self.output_templates(f, clinic) + + f_self = parameters[0] + selfless = parameters[1:] + assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!" + + if f.critical_section: + match len(f.target_critical_section): + case 0: + lock = 'Py_BEGIN_CRITICAL_SECTION({self_name});' + unlock = 'Py_END_CRITICAL_SECTION();' + case 1: + lock = 'Py_BEGIN_CRITICAL_SECTION({target_critical_section});' + unlock = 'Py_END_CRITICAL_SECTION();' + case _: + lock = 'Py_BEGIN_CRITICAL_SECTION2({target_critical_section});' + unlock = 'Py_END_CRITICAL_SECTION2();' + data.lock.append(lock) + data.unlock.append(unlock) + + last_group = 0 + first_optional = len(selfless) + positional = selfless and selfless[-1].is_positional_only() + has_option_groups = False + + # offset i by -1 because first_optional needs to ignore self + for i, p in enumerate(parameters, -1): + c = p.converter + + if (i != -1) and (p.default is not unspecified): + first_optional = min(first_optional, i) + + if p.is_vararg(): + data.cleanup.append(f"Py_XDECREF({c.parser_name});") + + # insert group variable + group = p.group + if last_group != group: + last_group = group + if group: + group_name = self.group_to_variable_name(group) + data.impl_arguments.append(group_name) + data.declarations.append("int " + group_name + " = 0;") + data.impl_parameters.append("int " + group_name) + has_option_groups = True + + c.render(p, data) + + if has_option_groups and (not positional): + fail("You cannot use optional groups ('[' and ']') " + "unless all parameters are positional-only ('/').") + + # HACK + # when we're METH_O, but have a custom return converter, + # we use "impl_parameters" for the parsing function + # because that works better. but that means we must + # suppress actually declaring the impl's parameters + # as variables in the parsing function. but since it's + # METH_O, we have exactly one anyway, so we know exactly + # where it is. + if ("METH_O" in templates['methoddef_define'] and + '{impl_parameters}' in templates['parser_prototype']): + data.declarations.pop(0) + + full_name = f.full_name + template_dict = {'full_name': full_name} + template_dict['name'] = f.displayname + if f.kind in {GETTER, SETTER}: + template_dict['getset_name'] = f.c_basename.upper() + template_dict['getset_basename'] = f.c_basename + if f.kind is GETTER: + template_dict['c_basename'] = f.c_basename + "_get" + elif f.kind is SETTER: + template_dict['c_basename'] = f.c_basename + "_set" + # Implicitly add the setter value parameter. + data.impl_parameters.append("PyObject *value") + data.impl_arguments.append("value") + else: + template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" + template_dict['c_basename'] = f.c_basename + + template_dict['docstring'] = libclinic.docstring_for_c_string(f.docstring) + template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' + template_dict['target_critical_section'] = ', '.join(f.target_critical_section) + for converter in converters: + converter.set_template_dict(template_dict) + + if f.kind not in {SETTER, METHOD_INIT}: + f.return_converter.render(f, data) + template_dict['impl_return_type'] = f.return_converter.type + + template_dict['declarations'] = libclinic.format_escape("\n".join(data.declarations)) + template_dict['initializers'] = "\n\n".join(data.initializers) + template_dict['modifications'] = '\n\n'.join(data.modifications) + template_dict['keywords_c'] = ' '.join('"' + k + '",' + for k in data.keywords) + keywords = [k for k in data.keywords if k] + template_dict['keywords_py'] = ' '.join('&_Py_ID(' + k + '),' + for k in keywords) + template_dict['format_units'] = ''.join(data.format_units) + template_dict['parse_arguments'] = ', '.join(data.parse_arguments) + if data.parse_arguments: + template_dict['parse_arguments_comma'] = ','; + else: + template_dict['parse_arguments_comma'] = ''; + template_dict['impl_parameters'] = ", ".join(data.impl_parameters) + template_dict['impl_arguments'] = ", ".join(data.impl_arguments) + + template_dict['return_conversion'] = libclinic.format_escape("".join(data.return_conversion).rstrip()) + template_dict['post_parsing'] = libclinic.format_escape("".join(data.post_parsing).rstrip()) + template_dict['cleanup'] = libclinic.format_escape("".join(data.cleanup)) + + template_dict['return_value'] = data.return_value + template_dict['lock'] = "\n".join(data.lock) + template_dict['unlock'] = "\n".join(data.unlock) + + # used by unpack tuple code generator + unpack_min = first_optional + unpack_max = len(selfless) + template_dict['unpack_min'] = str(unpack_min) + template_dict['unpack_max'] = str(unpack_max) + + if has_option_groups: + self.render_option_group_parsing(f, template_dict, + limited_capi=clinic.limited_capi) + + # buffers, not destination + for name, destination in clinic.destination_buffers.items(): + template = templates[name] + if has_option_groups: + template = libclinic.linear_format(template, + option_group_parsing=template_dict['option_group_parsing']) + template = libclinic.linear_format(template, + declarations=template_dict['declarations'], + return_conversion=template_dict['return_conversion'], + initializers=template_dict['initializers'], + modifications=template_dict['modifications'], + post_parsing=template_dict['post_parsing'], + cleanup=template_dict['cleanup'], + lock=template_dict['lock'], + unlock=template_dict['unlock'], + ) + + # Only generate the "exit:" label + # if we have any gotos + label = "exit:" if "goto exit;" in template else "" + template = libclinic.linear_format(template, exit_label=label) + + s = template.format_map(template_dict) + + # mild hack: + # reflow long impl declarations + if name in {"impl_prototype", "impl_definition"}: + s = libclinic.wrap_declarations(s) + + if clinic.line_prefix: + s = libclinic.indent_all_lines(s, clinic.line_prefix) + if clinic.line_suffix: + s = libclinic.suffix_all_lines(s, clinic.line_suffix) + + destination.append(s) + + return clinic.get_destination('block').dump() diff --git a/Tools/clinic/libclinic/codegen.py b/Tools/clinic/libclinic/codegen.py new file mode 100644 index 00000000000000..097fff0edf38c6 --- /dev/null +++ b/Tools/clinic/libclinic/codegen.py @@ -0,0 +1,187 @@ +from __future__ import annotations +import dataclasses as dc +import io +import os +from typing import Final, TYPE_CHECKING +if TYPE_CHECKING: + from clinic import Clinic + +import libclinic +from libclinic import fail +from libclinic.crenderdata import Include +from libclinic.language import Language +from libclinic.block_parser import Block + + +@dc.dataclass(slots=True) +class BlockPrinter: + language: Language + f: io.StringIO = dc.field(default_factory=io.StringIO) + + # '#include "header.h" // reason': column of '//' comment + INCLUDE_COMMENT_COLUMN: Final[int] = 35 + + def print_block( + self, + block: Block, + *, + core_includes: bool = False, + limited_capi: bool, + header_includes: dict[str, Include], + ) -> None: + input = block.input + output = block.output + dsl_name = block.dsl_name + write = self.f.write + + assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name) + + if not dsl_name: + write(input) + return + + write(self.language.start_line.format(dsl_name=dsl_name)) + write("\n") + + body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) + if not body_prefix: + write(input) + else: + for line in input.split('\n'): + write(body_prefix) + write(line) + write("\n") + + write(self.language.stop_line.format(dsl_name=dsl_name)) + write("\n") + + output = '' + if core_includes and header_includes: + # Emit optional "#include" directives for C headers + output += '\n' + + current_condition: str | None = None + includes = sorted(header_includes.values(), key=Include.sort_key) + for include in includes: + if include.condition != current_condition: + if current_condition: + output += '#endif\n' + current_condition = include.condition + if include.condition: + output += f'{include.condition}\n' + + if current_condition: + line = f'# include "{include.filename}"' + else: + line = f'#include "{include.filename}"' + if include.reason: + comment = f'// {include.reason}\n' + line = line.ljust(self.INCLUDE_COMMENT_COLUMN - 1) + comment + output += line + + if current_condition: + output += '#endif\n' + + input = ''.join(block.input) + output += ''.join(block.output) + if output: + if not output.endswith('\n'): + output += '\n' + write(output) + + arguments = "output={output} input={input}".format( + output=libclinic.compute_checksum(output, 16), + input=libclinic.compute_checksum(input, 16) + ) + write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) + write("\n") + + def write(self, text: str) -> None: + self.f.write(text) + + +class BufferSeries: + """ + Behaves like a "defaultlist". + When you ask for an index that doesn't exist yet, + the object grows the list until that item exists. + So o[n] will always work. + + Supports negative indices for actual items. + e.g. o[-1] is an element immediately preceding o[0]. + """ + + def __init__(self) -> None: + self._start = 0 + self._array: list[list[str]] = [] + + def __getitem__(self, i: int) -> list[str]: + i -= self._start + if i < 0: + self._start += i + prefix: list[list[str]] = [[] for x in range(-i)] + self._array = prefix + self._array + i = 0 + while i >= len(self._array): + self._array.append([]) + return self._array[i] + + def clear(self) -> None: + for ta in self._array: + ta.clear() + + def dump(self) -> str: + texts = ["".join(ta) for ta in self._array] + self.clear() + return "".join(texts) + + +@dc.dataclass(slots=True, repr=False) +class Destination: + name: str + type: str + clinic: Clinic + buffers: BufferSeries = dc.field(init=False, default_factory=BufferSeries) + filename: str = dc.field(init=False) # set in __post_init__ + + args: dc.InitVar[tuple[str, ...]] = () + + def __post_init__(self, args: tuple[str, ...]) -> None: + valid_types = ('buffer', 'file', 'suppress') + if self.type not in valid_types: + fail( + f"Invalid destination type {self.type!r} for {self.name}, " + f"must be {', '.join(valid_types)}" + ) + extra_arguments = 1 if self.type == "file" else 0 + if len(args) < extra_arguments: + fail(f"Not enough arguments for destination " + f"{self.name!r} new {self.type!r}") + if len(args) > extra_arguments: + fail(f"Too many arguments for destination {self.name!r} new {self.type!r}") + if self.type =='file': + d = {} + filename = self.clinic.filename + d['path'] = filename + dirname, basename = os.path.split(filename) + if not dirname: + dirname = '.' + d['dirname'] = dirname + d['basename'] = basename + d['basename_root'], d['basename_extension'] = os.path.splitext(filename) + self.filename = args[0].format_map(d) + + def __repr__(self) -> str: + if self.type == 'file': + type_repr = f"type='file' file={self.filename!r}" + else: + type_repr = f"type={self.type!r}" + return f"" + + def clear(self) -> None: + if self.type != 'buffer': + fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'") + self.buffers.clear() + + def dump(self) -> str: + return self.buffers.dump() diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index 1bfaad00cd0f08..1beed13b437886 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -4,6 +4,7 @@ import enum import functools import inspect +from collections.abc import Iterable, Iterator, Sequence from typing import Final, Any, TYPE_CHECKING if TYPE_CHECKING: from clinic import Clinic @@ -238,3 +239,73 @@ def render_docstring(self) -> str: lines = [f" {self.name}"] lines.extend(f" {line}" for line in self.docstring.split("\n")) return "\n".join(lines).rstrip() + + +ParamTuple = tuple["Parameter", ...] + + +def permute_left_option_groups( + l: Sequence[Iterable[Parameter]] +) -> Iterator[ParamTuple]: + """ + Given [(1,), (2,), (3,)], should yield: + () + (3,) + (2, 3) + (1, 2, 3) + """ + yield tuple() + accumulator: list[Parameter] = [] + for group in reversed(l): + accumulator = list(group) + accumulator + yield tuple(accumulator) + + +def permute_right_option_groups( + l: Sequence[Iterable[Parameter]] +) -> Iterator[ParamTuple]: + """ + Given [(1,), (2,), (3,)], should yield: + () + (1,) + (1, 2) + (1, 2, 3) + """ + yield tuple() + accumulator: list[Parameter] = [] + for group in l: + accumulator.extend(group) + yield tuple(accumulator) + + +def permute_optional_groups( + left: Sequence[Iterable[Parameter]], + required: Iterable[Parameter], + right: Sequence[Iterable[Parameter]] +) -> tuple[ParamTuple, ...]: + """ + Generator function that computes the set of acceptable + argument lists for the provided iterables of + argument groups. (Actually it generates a tuple of tuples.) + + Algorithm: prefer left options over right options. + + If required is empty, left must also be empty. + """ + required = tuple(required) + if not required: + if left: + raise ValueError("required is empty but left is not") + + accumulator: list[ParamTuple] = [] + counts = set() + for r in permute_right_option_groups(right): + for l in permute_left_option_groups(left): + t = l + required + r + if len(t) in counts: + continue + counts.add(len(t)) + accumulator.append(t) + + accumulator.sort(key=len) + return tuple(accumulator) From 2057c92125f2e37caee209f032be9fe9c208357b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 3 Apr 2024 23:02:42 +0200 Subject: [PATCH 073/143] gh-114329: Fix PyList_GetItemRef() limited C API definition (#117520) --- Include/listobject.h | 2 ++ Modules/_testlimitedcapi/heaptype_relative.c | 2 +- Modules/_testlimitedcapi/list.c | 6 ++++++ Modules/_testlimitedcapi/object.c | 2 +- Modules/_testlimitedcapi/parts.h | 2 +- Modules/_testlimitedcapi/vectorcall_limited.c | 2 +- 6 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Include/listobject.h b/Include/listobject.h index 4e4084b43483a2..e1e059b0ba7466 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -29,7 +29,9 @@ PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size); PyAPI_FUNC(Py_ssize_t) PyList_Size(PyObject *); PyAPI_FUNC(PyObject *) PyList_GetItem(PyObject *, Py_ssize_t); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 PyAPI_FUNC(PyObject *) PyList_GetItemRef(PyObject *, Py_ssize_t); +#endif PyAPI_FUNC(int) PyList_SetItem(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Insert(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Append(PyObject *, PyObject *); diff --git a/Modules/_testlimitedcapi/heaptype_relative.c b/Modules/_testlimitedcapi/heaptype_relative.c index 7c508c6182bc8a..c2531518d86a51 100644 --- a/Modules/_testlimitedcapi/heaptype_relative.c +++ b/Modules/_testlimitedcapi/heaptype_relative.c @@ -1,6 +1,6 @@ // Need limited C API version 3.12 for PyType_FromMetaclass() #include "pyconfig.h" // Py_GIL_DISABLED -#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) # define Py_LIMITED_API 0x030c0000 #endif diff --git a/Modules/_testlimitedcapi/list.c b/Modules/_testlimitedcapi/list.c index 3022cbf9191b2e..ed492c3e719727 100644 --- a/Modules/_testlimitedcapi/list.c +++ b/Modules/_testlimitedcapi/list.c @@ -1,3 +1,9 @@ +// Need limited C API version 3.13 for PyList_GetItemRef() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030d0000 +#endif + #include "parts.h" #include "util.h" diff --git a/Modules/_testlimitedcapi/object.c b/Modules/_testlimitedcapi/object.c index 6e438c811d6e98..da6fe3e4efa34c 100644 --- a/Modules/_testlimitedcapi/object.c +++ b/Modules/_testlimitedcapi/object.c @@ -1,6 +1,6 @@ // Need limited C API version 3.13 for Py_GetConstant() #include "pyconfig.h" // Py_GIL_DISABLED -#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) # define Py_LIMITED_API 0x030d0000 #endif diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index d91f174cd31eed..d5e590a8dcd679 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -7,7 +7,7 @@ #include "pyconfig.h" // Py_GIL_DISABLED // Use the limited C API -#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) // need limited C API version 3.5 for PyModule_AddFunctions() # define Py_LIMITED_API 0x03050000 #endif diff --git a/Modules/_testlimitedcapi/vectorcall_limited.c b/Modules/_testlimitedcapi/vectorcall_limited.c index 784126c17fccc1..5ef97ca8a063e1 100644 --- a/Modules/_testlimitedcapi/vectorcall_limited.c +++ b/Modules/_testlimitedcapi/vectorcall_limited.c @@ -2,7 +2,7 @@ // Need limited C API version 3.12 for PyObject_Vectorcall() #include "pyconfig.h" // Py_GIL_DISABLED -#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) # define Py_LIMITED_API 0x030c0000 #endif From 985917dc8d34e2d2f717f7a981580a8dcf18d53a Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 3 Apr 2024 23:14:55 +0100 Subject: [PATCH 074/143] gh-117267: Ensure DirEntry.stat().st_ctime still contains creation time during deprecation period (GH-117354) --- .../Windows/2024-03-28-22-12-00.gh-issue-117267.K_tki1.rst | 5 +++++ Modules/posixmodule.c | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2024-03-28-22-12-00.gh-issue-117267.K_tki1.rst diff --git a/Misc/NEWS.d/next/Windows/2024-03-28-22-12-00.gh-issue-117267.K_tki1.rst b/Misc/NEWS.d/next/Windows/2024-03-28-22-12-00.gh-issue-117267.K_tki1.rst new file mode 100644 index 00000000000000..d3221429850a11 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-03-28-22-12-00.gh-issue-117267.K_tki1.rst @@ -0,0 +1,5 @@ +Ensure ``DirEntry.stat().st_ctime`` behaves consistently with +:func:`os.stat` during the deprecation period of ``st_ctime`` by containing +the same value as ``st_birthtime``. After the deprecation period, +``st_ctime`` will be the metadata change time (or unavailable through +``DirEntry``), and only ``st_birthtime`` will contain the creation time. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index fcac3dbe3553ef..5e54cf64cd563e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -15830,6 +15830,10 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) find_data_to_file_info(dataW, &file_info, &reparse_tag); _Py_attribute_data_to_stat(&file_info, reparse_tag, NULL, NULL, &entry->win32_lstat); + /* ctime is only deprecated from 3.12, so we copy birthtime across */ + entry->win32_lstat.st_ctime = entry->win32_lstat.st_birthtime; + entry->win32_lstat.st_ctime_nsec = entry->win32_lstat.st_birthtime_nsec; + return (PyObject *)entry; error: From b4fe02f595fcb9f78261920a268ef614821ec195 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:24:24 -0700 Subject: [PATCH 075/143] gh-117205: Increase chunksize when compiling pyc in parallel (#117206) --- Lib/compileall.py | 3 ++- .../Library/2024-03-25-00-20-16.gh-issue-117205.yV7xGb.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-25-00-20-16.gh-issue-117205.yV7xGb.rst diff --git a/Lib/compileall.py b/Lib/compileall.py index 9b53086bf41380..47e2446356e7d7 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -116,7 +116,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, prependdir=prependdir, limit_sl_dest=limit_sl_dest, hardlink_dupes=hardlink_dupes), - files) + files, + chunksize=4) success = min(results, default=True) else: for file in files: diff --git a/Misc/NEWS.d/next/Library/2024-03-25-00-20-16.gh-issue-117205.yV7xGb.rst b/Misc/NEWS.d/next/Library/2024-03-25-00-20-16.gh-issue-117205.yV7xGb.rst new file mode 100644 index 00000000000000..8d8c201afd29fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-25-00-20-16.gh-issue-117205.yV7xGb.rst @@ -0,0 +1 @@ +Speed up :func:`compileall.compile_dir` by 20% when using multiprocessing by increasing ``chunksize``. From 85843348c5f0b8c2f973e8bc586475e69af19cd2 Mon Sep 17 00:00:00 2001 From: rsp4jack Date: Thu, 4 Apr 2024 11:13:32 +0800 Subject: [PATCH 076/143] gh-117459: Keep the traceback in _convert_future_exc (#117460) --- Lib/asyncio/futures.py | 6 ++---- Lib/test/test_asyncio/test_futures.py | 19 +++++++++++++++++++ ...-04-02-13-13-46.gh-issue-117459.jiIZmH.rst | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-02-13-13-46.gh-issue-117459.jiIZmH.rst diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 5d35321db7943b..9c1b5e49e1a70b 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -319,11 +319,9 @@ def _set_result_unless_cancelled(fut, result): def _convert_future_exc(exc): exc_class = type(exc) if exc_class is concurrent.futures.CancelledError: - return exceptions.CancelledError(*exc.args) - elif exc_class is concurrent.futures.TimeoutError: - return exceptions.TimeoutError(*exc.args) + return exceptions.CancelledError(*exc.args).with_traceback(exc.__traceback__) elif exc_class is concurrent.futures.InvalidStateError: - return exceptions.InvalidStateError(*exc.args) + return exceptions.InvalidStateError(*exc.args).with_traceback(exc.__traceback__) else: return exc diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index d3e8efec1c04c2..458b70451a306a 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -5,6 +5,7 @@ import re import sys import threading +import traceback import unittest from unittest import mock from types import GenericAlias @@ -416,6 +417,24 @@ def test_copy_state(self): _copy_future_state(f_cancelled, newf_cancelled) self.assertTrue(newf_cancelled.cancelled()) + try: + raise concurrent.futures.InvalidStateError + except BaseException as e: + f_exc = e + + f_conexc = self._new_future(loop=self.loop) + f_conexc.set_exception(f_exc) + + newf_conexc = self._new_future(loop=self.loop) + _copy_future_state(f_conexc, newf_conexc) + self.assertTrue(newf_conexc.done()) + try: + newf_conexc.result() + except BaseException as e: + newf_exc = e # assertRaises context manager drops the traceback + newf_tb = ''.join(traceback.format_tb(newf_exc.__traceback__)) + self.assertEqual(newf_tb.count('raise concurrent.futures.InvalidStateError'), 1) + def test_iter(self): fut = self._new_future(loop=self.loop) diff --git a/Misc/NEWS.d/next/Library/2024-04-02-13-13-46.gh-issue-117459.jiIZmH.rst b/Misc/NEWS.d/next/Library/2024-04-02-13-13-46.gh-issue-117459.jiIZmH.rst new file mode 100644 index 00000000000000..549bd44112befe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-02-13-13-46.gh-issue-117459.jiIZmH.rst @@ -0,0 +1 @@ +:meth:`asyncio.asyncio.run_coroutine_threadsafe` now keeps the traceback of :class:`CancelledError`, :class:`TimeoutError` and :class:`InvalidStateError` which are raised in the coroutine. From dc5471404489da53e6d591b52ba8886897ed3743 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 4 Apr 2024 11:09:40 +0200 Subject: [PATCH 077/143] gh-113317: Finish splitting Argument Clinic into sub-files (#117513) Add libclinic.parser module and move the following classes and functions there: * Parser * PythonParser * create_parser_namespace() Add libclinic.dsl_parser module and move the following classes, functions and variables there: * ConverterArgs * DSLParser * FunctionNames * IndentStack * ParamState * StateKeeper * eval_ast_expr() * unsupported_special_methods Add libclinic.app module and move the Clinic class there. Add libclinic.cli module and move the following functions there: * create_cli() * main() * parse_file() * run_clinic() --- Lib/test/test_clinic.py | 104 +- Tools/clinic/clinic.py | 2142 +------------------------- Tools/clinic/libclinic/app.py | 297 ++++ Tools/clinic/libclinic/clanguage.py | 2 +- Tools/clinic/libclinic/cli.py | 231 +++ Tools/clinic/libclinic/codegen.py | 7 +- Tools/clinic/libclinic/dsl_parser.py | 1592 +++++++++++++++++++ Tools/clinic/libclinic/function.py | 2 +- Tools/clinic/libclinic/language.py | 2 +- Tools/clinic/libclinic/parser.py | 53 + 10 files changed, 2238 insertions(+), 2194 deletions(-) create mode 100644 Tools/clinic/libclinic/app.py create mode 100644 Tools/clinic/libclinic/cli.py create mode 100644 Tools/clinic/libclinic/dsl_parser.py create mode 100644 Tools/clinic/libclinic/parser.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index df8b3d261c4278..9788ac0261fa49 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -17,18 +17,26 @@ test_tools.skip_if_missing('clinic') with test_tools.imports_under_tool('clinic'): import libclinic - from libclinic.converters import int_converter, str_converter + from libclinic import ClinicError, unspecified, NULL, fail + from libclinic.converters import int_converter, str_converter, self_converter from libclinic.function import ( + Module, Class, Function, FunctionKind, Parameter, permute_optional_groups, permute_right_option_groups, permute_left_option_groups) import clinic - from clinic import DSLParser + from libclinic.clanguage import CLanguage + from libclinic.converter import converters, legacy_converters + from libclinic.return_converters import return_converters, int_return_converter + from libclinic.block_parser import Block, BlockParser + from libclinic.codegen import BlockPrinter, Destination + from libclinic.dsl_parser import DSLParser + from libclinic.cli import parse_file, Clinic def _make_clinic(*, filename='clinic_tests', limited_capi=False): - clang = clinic.CLanguage(filename) - c = clinic.Clinic(clang, filename=filename, limited_capi=limited_capi) - c.block_parser = clinic.BlockParser('', clang) + clang = CLanguage(filename) + c = Clinic(clang, filename=filename, limited_capi=limited_capi) + c.block_parser = BlockParser('', clang) return c @@ -47,7 +55,7 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, if strip: code = code.strip() errmsg = re.escape(errmsg) - with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + with tc.assertRaisesRegex(ClinicError, errmsg) as cm: parser(code) if filename is not None: tc.assertEqual(cm.exception.filename, filename) @@ -62,12 +70,12 @@ def restore_dict(converters, old_converters): def save_restore_converters(testcase): - testcase.addCleanup(restore_dict, clinic.converters, - clinic.converters.copy()) - testcase.addCleanup(restore_dict, clinic.legacy_converters, - clinic.legacy_converters.copy()) - testcase.addCleanup(restore_dict, clinic.return_converters, - clinic.return_converters.copy()) + testcase.addCleanup(restore_dict, converters, + converters.copy()) + testcase.addCleanup(restore_dict, legacy_converters, + legacy_converters.copy()) + testcase.addCleanup(restore_dict, return_converters, + return_converters.copy()) class ClinicWholeFileTest(TestCase): @@ -140,11 +148,11 @@ def test_whitespace_before_stop_line(self): self.expect_failure(raw, err, filename="test.c", lineno=2) def test_parse_with_body_prefix(self): - clang = clinic.CLanguage(None) + clang = CLanguage(None) clang.body_prefix = "//" clang.start_line = "//[{dsl_name} start]" clang.stop_line = "//[{dsl_name} stop]" - cl = clinic.Clinic(clang, filename="test.c", limited_capi=False) + cl = Clinic(clang, filename="test.c", limited_capi=False) raw = dedent(""" //[clinic start] //module test @@ -660,8 +668,8 @@ def expect_parsing_failure( self, *, filename, expected_error, verify=True, output=None ): errmsg = re.escape(dedent(expected_error).strip()) - with self.assertRaisesRegex(clinic.ClinicError, errmsg): - clinic.parse_file(filename, limited_capi=False) + with self.assertRaisesRegex(ClinicError, errmsg): + parse_file(filename, limited_capi=False) def test_parse_file_no_extension(self) -> None: self.expect_parsing_failure( @@ -782,13 +790,13 @@ def test_multiline_substitution(self): def test_text_before_block_marker(self): regex = re.escape("found before '{marker}'") - with self.assertRaisesRegex(clinic.ClinicError, regex): + with self.assertRaisesRegex(ClinicError, regex): libclinic.linear_format("no text before marker for you! {marker}", marker="not allowed!") def test_text_after_block_marker(self): regex = re.escape("found after '{marker}'") - with self.assertRaisesRegex(clinic.ClinicError, regex): + with self.assertRaisesRegex(ClinicError, regex): libclinic.linear_format("{marker} no text after marker for you!", marker="not allowed!") @@ -810,10 +818,10 @@ def parse(self, block): class ClinicBlockParserTest(TestCase): def _test(self, input, output): - language = clinic.CLanguage(None) + language = CLanguage(None) - blocks = list(clinic.BlockParser(input, language)) - writer = clinic.BlockPrinter(language) + blocks = list(BlockParser(input, language)) + writer = BlockPrinter(language) c = _make_clinic() for block in blocks: writer.print_block(block, limited_capi=c.limited_capi, header_includes=c.includes) @@ -841,8 +849,8 @@ def test_round_trip_2(self): """) def _test_clinic(self, input, output): - language = clinic.CLanguage(None) - c = clinic.Clinic(language, filename="file", limited_capi=False) + language = CLanguage(None) + c = Clinic(language, filename="file", limited_capi=False) c.parsers['inert'] = InertParser(c) c.parsers['copy'] = CopyParser(c) computed = c.parse(input) @@ -875,7 +883,7 @@ class ClinicParserTest(TestCase): def parse(self, text): c = _make_clinic() parser = DSLParser(c) - block = clinic.Block(text) + block = Block(text) parser.parse(block) return block @@ -883,8 +891,8 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): block = self.parse(text) s = block.signatures self.assertEqual(len(s), signatures_in_block) - assert isinstance(s[0], clinic.Module) - assert isinstance(s[function_index], clinic.Function) + assert isinstance(s[0], Module) + assert isinstance(s[function_index], Function) return s[function_index] def expect_failure(self, block, err, *, @@ -899,7 +907,7 @@ def checkDocstring(self, fn, expected): def test_trivial(self): parser = DSLParser(_make_clinic()) - block = clinic.Block(""" + block = Block(""" module os os.access """) @@ -1188,7 +1196,7 @@ def test_base_invalid_syntax(self): Function 'stat' has an invalid parameter declaration: \s+'invalid syntax: int = 42' """).strip() - with self.assertRaisesRegex(clinic.ClinicError, err): + with self.assertRaisesRegex(ClinicError, err): self.parse_function(block) def test_param_default_invalid_syntax(self): @@ -1220,7 +1228,7 @@ def test_return_converter(self): module os os.stat -> int """) - self.assertIsInstance(function.return_converter, clinic.int_return_converter) + self.assertIsInstance(function.return_converter, int_return_converter) def test_return_converter_invalid_syntax(self): block = """ @@ -2036,7 +2044,7 @@ def test_directive(self): parser = DSLParser(_make_clinic()) parser.flag = False parser.directives['setflag'] = lambda : setattr(parser, 'flag', True) - block = clinic.Block("setflag") + block = Block("setflag") parser.parse(block) self.assertTrue(parser.flag) @@ -2301,14 +2309,14 @@ def test_unused_param(self): def test_scaffolding(self): # test repr on special values - self.assertEqual(repr(clinic.unspecified), '') - self.assertEqual(repr(clinic.NULL), '') + self.assertEqual(repr(unspecified), '') + self.assertEqual(repr(NULL), '') # test that fail fails with support.captured_stdout() as stdout: errmsg = 'The igloos are melting' - with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: - clinic.fail(errmsg, filename='clown.txt', line_number=69) + with self.assertRaisesRegex(ClinicError, errmsg) as cm: + fail(errmsg, filename='clown.txt', line_number=69) exc = cm.exception self.assertEqual(exc.filename, 'clown.txt') self.assertEqual(exc.lineno, 69) @@ -3998,15 +4006,15 @@ def test_suffix_all_lines(self): class ClinicReprTests(unittest.TestCase): def test_Block_repr(self): - block = clinic.Block("foo") + block = Block("foo") expected_repr = "" self.assertEqual(repr(block), expected_repr) - block2 = clinic.Block("bar", "baz", [], "eggs", "spam") + block2 = Block("bar", "baz", [], "eggs", "spam") expected_repr_2 = "" self.assertEqual(repr(block2), expected_repr_2) - block3 = clinic.Block( + block3 = Block( input="longboi_" * 100, dsl_name="wow_so_long", signatures=[], @@ -4021,47 +4029,47 @@ def test_Block_repr(self): def test_Destination_repr(self): c = _make_clinic() - destination = clinic.Destination( + destination = Destination( "foo", type="file", clinic=c, args=("eggs",) ) self.assertEqual( repr(destination), "" ) - destination2 = clinic.Destination("bar", type="buffer", clinic=c) + destination2 = Destination("bar", type="buffer", clinic=c) self.assertEqual(repr(destination2), "") def test_Module_repr(self): - module = clinic.Module("foo", _make_clinic()) + module = Module("foo", _make_clinic()) self.assertRegex(repr(module), r"") def test_Class_repr(self): - cls = clinic.Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') + cls = Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') self.assertRegex(repr(cls), r"") def test_FunctionKind_repr(self): self.assertEqual( - repr(clinic.FunctionKind.INVALID), "" + repr(FunctionKind.INVALID), "" ) self.assertEqual( - repr(clinic.FunctionKind.CLASS_METHOD), "" + repr(FunctionKind.CLASS_METHOD), "" ) def test_Function_and_Parameter_reprs(self): - function = clinic.Function( + function = Function( name='foo', module=_make_clinic(), cls=None, c_basename=None, full_name='foofoo', - return_converter=clinic.int_return_converter(), - kind=clinic.FunctionKind.METHOD_INIT, + return_converter=int_return_converter(), + kind=FunctionKind.METHOD_INIT, coexist=False ) self.assertEqual(repr(function), "") - converter = clinic.self_converter('bar', 'bar', function) - parameter = clinic.Parameter( + converter = self_converter('bar', 'bar', function) + parameter = Parameter( "bar", kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, function=function, diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index d8043128bf3d01..362715d264620e 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4,2147 +4,7 @@ # Copyright 2012-2013 by Larry Hastings. # Licensed to the PSF under a contributor agreement. # -from __future__ import annotations - -import argparse -import ast -import contextlib -import enum -import functools -import inspect -import io -import os -import pprint -import re -import shlex -import sys - -from collections.abc import ( - Callable, - Sequence, -) -from types import FunctionType, NoneType -from typing import ( - Any, - NamedTuple, - NoReturn, - Protocol, -) - - -# Local imports. -import libclinic -import libclinic.cpp -from libclinic import ( - ClinicError, VersionTuple, - fail, warn, unspecified, unknown, NULL) -from libclinic.function import ( - Module, Class, Function, Parameter, - ClassDict, ModuleDict, FunctionKind, - CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, - GETTER, SETTER) -from libclinic.language import Language, PythonLanguage -from libclinic.block_parser import Block, BlockParser -from libclinic.crenderdata import Include -from libclinic.converter import ( - CConverter, ConverterType, - converters, legacy_converters) -from libclinic.converters import ( - self_converter, defining_class_converter, buffer, - robuffer, rwbuffer, correct_name_for_self) -from libclinic.return_converters import ( - CReturnConverter, return_converters, - int_return_converter, ReturnConverterType) -from libclinic.clanguage import CLanguage -from libclinic.codegen import BlockPrinter, Destination - - -# TODO: -# -# soon: -# -# * allow mixing any two of {positional-only, positional-or-keyword, -# keyword-only} -# * dict constructor uses positional-only and keyword-only -# * max and min use positional only with an optional group -# and keyword-only -# - - -# Match '#define Py_LIMITED_API'. -# Match '# define Py_LIMITED_API 0x030d0000' (without the version). -LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API') - - -# "extensions" maps the file extension ("c", "py") to Language classes. -LangDict = dict[str, Callable[[str], Language]] -extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } -extensions['py'] = PythonLanguage - - -DestinationDict = dict[str, Destination] - - -class Parser(Protocol): - def __init__(self, clinic: Clinic) -> None: ... - def parse(self, block: Block) -> None: ... - - -class Clinic: - - presets_text = """ -preset block -everything block -methoddef_ifndef buffer 1 -docstring_prototype suppress -parser_prototype suppress -cpp_if suppress -cpp_endif suppress - -preset original -everything block -methoddef_ifndef buffer 1 -docstring_prototype suppress -parser_prototype suppress -cpp_if suppress -cpp_endif suppress - -preset file -everything file -methoddef_ifndef file 1 -docstring_prototype suppress -parser_prototype suppress -impl_definition block - -preset buffer -everything buffer -methoddef_ifndef buffer 1 -impl_definition block -docstring_prototype suppress -impl_prototype suppress -parser_prototype suppress - -preset partial-buffer -everything buffer -methoddef_ifndef buffer 1 -docstring_prototype block -impl_prototype suppress -methoddef_define block -parser_prototype block -impl_definition block - -""" - - def __init__( - self, - language: CLanguage, - printer: BlockPrinter | None = None, - *, - filename: str, - limited_capi: bool, - verify: bool = True, - ) -> None: - # maps strings to Parser objects. - # (instantiated from the "parsers" global.) - self.parsers: dict[str, Parser] = {} - self.language: CLanguage = language - if printer: - fail("Custom printers are broken right now") - self.printer = printer or BlockPrinter(language) - self.verify = verify - self.limited_capi = limited_capi - self.filename = filename - self.modules: ModuleDict = {} - self.classes: ClassDict = {} - self.functions: list[Function] = [] - # dict: include name => Include instance - self.includes: dict[str, Include] = {} - - self.line_prefix = self.line_suffix = '' - - self.destinations: DestinationDict = {} - self.add_destination("block", "buffer") - self.add_destination("suppress", "suppress") - self.add_destination("buffer", "buffer") - if filename: - self.add_destination("file", "file", "{dirname}/clinic/{basename}.h") - - d = self.get_destination_buffer - self.destination_buffers = { - 'cpp_if': d('file'), - 'docstring_prototype': d('suppress'), - 'docstring_definition': d('file'), - 'methoddef_define': d('file'), - 'impl_prototype': d('file'), - 'parser_prototype': d('suppress'), - 'parser_definition': d('file'), - 'cpp_endif': d('file'), - 'methoddef_ifndef': d('file', 1), - 'impl_definition': d('block'), - } - - DestBufferType = dict[str, list[str]] - DestBufferList = list[DestBufferType] - - self.destination_buffers_stack: DestBufferList = [] - self.ifndef_symbols: set[str] = set() - - self.presets: dict[str, dict[Any, Any]] = {} - preset = None - for line in self.presets_text.strip().split('\n'): - line = line.strip() - if not line: - continue - name, value, *options = line.split() - if name == 'preset': - self.presets[value] = preset = {} - continue - - if len(options): - index = int(options[0]) - else: - index = 0 - buffer = self.get_destination_buffer(value, index) - - if name == 'everything': - for name in self.destination_buffers: - preset[name] = buffer - continue - - assert name in self.destination_buffers - preset[name] = buffer - - def add_include(self, name: str, reason: str, - *, condition: str | None = None) -> None: - try: - existing = self.includes[name] - except KeyError: - pass - else: - if existing.condition and not condition: - # If the previous include has a condition and the new one is - # unconditional, override the include. - pass - else: - # Already included, do nothing. Only mention a single reason, - # no need to list all of them. - return - - self.includes[name] = Include(name, reason, condition) - - def add_destination( - self, - name: str, - type: str, - *args: str - ) -> None: - if name in self.destinations: - fail(f"Destination already exists: {name!r}") - self.destinations[name] = Destination(name, type, self, args) - - def get_destination(self, name: str) -> Destination: - d = self.destinations.get(name) - if not d: - fail(f"Destination does not exist: {name!r}") - return d - - def get_destination_buffer( - self, - name: str, - item: int = 0 - ) -> list[str]: - d = self.get_destination(name) - return d.buffers[item] - - def parse(self, input: str) -> str: - printer = self.printer - self.block_parser = BlockParser(input, self.language, verify=self.verify) - for block in self.block_parser: - dsl_name = block.dsl_name - if dsl_name: - if dsl_name not in self.parsers: - assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." - self.parsers[dsl_name] = parsers[dsl_name](self) - parser = self.parsers[dsl_name] - parser.parse(block) - printer.print_block(block, - limited_capi=self.limited_capi, - header_includes=self.includes) - - # these are destinations not buffers - for name, destination in self.destinations.items(): - if destination.type == 'suppress': - continue - output = destination.dump() - - if output: - block = Block("", dsl_name="clinic", output=output) - - if destination.type == 'buffer': - block.input = "dump " + name + "\n" - warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") - printer.write("\n") - printer.print_block(block, - limited_capi=self.limited_capi, - header_includes=self.includes) - continue - - if destination.type == 'file': - try: - dirname = os.path.dirname(destination.filename) - try: - os.makedirs(dirname) - except FileExistsError: - if not os.path.isdir(dirname): - fail(f"Can't write to destination " - f"{destination.filename!r}; " - f"can't make directory {dirname!r}!") - if self.verify: - with open(destination.filename) as f: - parser_2 = BlockParser(f.read(), language=self.language) - blocks = list(parser_2) - if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): - fail(f"Modified destination file " - f"{destination.filename!r}; not overwriting!") - except FileNotFoundError: - pass - - block.input = 'preserve\n' - printer_2 = BlockPrinter(self.language) - printer_2.print_block(block, - core_includes=True, - limited_capi=self.limited_capi, - header_includes=self.includes) - libclinic.write_file(destination.filename, - printer_2.f.getvalue()) - continue - - return printer.f.getvalue() - - def _module_and_class( - self, fields: Sequence[str] - ) -> tuple[Module | Clinic, Class | None]: - """ - fields should be an iterable of field names. - returns a tuple of (module, class). - the module object could actually be self (a clinic object). - this function is only ever used to find the parent of where - a new class/module should go. - """ - parent: Clinic | Module | Class = self - module: Clinic | Module = self - cls: Class | None = None - - for idx, field in enumerate(fields): - if not isinstance(parent, Class): - if field in parent.modules: - parent = module = parent.modules[field] - continue - if field in parent.classes: - parent = cls = parent.classes[field] - else: - fullname = ".".join(fields[idx:]) - fail(f"Parent class or module {fullname!r} does not exist.") - - return module, cls - - def __repr__(self) -> str: - return "" - - -def parse_file( - filename: str, - *, - limited_capi: bool, - output: str | None = None, - verify: bool = True, -) -> None: - if not output: - output = filename - - extension = os.path.splitext(filename)[1][1:] - if not extension: - raise ClinicError(f"Can't extract file type for file {filename!r}") - - try: - language = extensions[extension](filename) - except KeyError: - raise ClinicError(f"Can't identify file type for file {filename!r}") - - with open(filename, encoding="utf-8") as f: - raw = f.read() - - # exit quickly if there are no clinic markers in the file - find_start_re = BlockParser("", language).find_start_re - if not find_start_re.search(raw): - return - - if LIMITED_CAPI_REGEX.search(raw): - limited_capi = True - - assert isinstance(language, CLanguage) - clinic = Clinic(language, - verify=verify, - filename=filename, - limited_capi=limited_capi) - cooked = clinic.parse(raw) - - libclinic.write_file(output, cooked) - - -@functools.cache -def _create_parser_base_namespace() -> dict[str, Any]: - ns = dict( - CConverter=CConverter, - CReturnConverter=CReturnConverter, - buffer=buffer, - robuffer=robuffer, - rwbuffer=rwbuffer, - unspecified=unspecified, - NoneType=NoneType, - ) - for name, converter in converters.items(): - ns[f'{name}_converter'] = converter - for name, return_converter in return_converters.items(): - ns[f'{name}_return_converter'] = return_converter - return ns - - -def create_parser_namespace() -> dict[str, Any]: - base_namespace = _create_parser_base_namespace() - return base_namespace.copy() - - - -class PythonParser: - def __init__(self, clinic: Clinic) -> None: - pass - - def parse(self, block: Block) -> None: - namespace = create_parser_namespace() - with contextlib.redirect_stdout(io.StringIO()) as s: - exec(block.input, namespace) - block.output = s.getvalue() - - -unsupported_special_methods: set[str] = set(""" - -__abs__ -__add__ -__and__ -__call__ -__delitem__ -__divmod__ -__eq__ -__float__ -__floordiv__ -__ge__ -__getattr__ -__getattribute__ -__getitem__ -__gt__ -__hash__ -__iadd__ -__iand__ -__ifloordiv__ -__ilshift__ -__imatmul__ -__imod__ -__imul__ -__index__ -__int__ -__invert__ -__ior__ -__ipow__ -__irshift__ -__isub__ -__iter__ -__itruediv__ -__ixor__ -__le__ -__len__ -__lshift__ -__lt__ -__matmul__ -__mod__ -__mul__ -__neg__ -__next__ -__or__ -__pos__ -__pow__ -__radd__ -__rand__ -__rdivmod__ -__repr__ -__rfloordiv__ -__rlshift__ -__rmatmul__ -__rmod__ -__rmul__ -__ror__ -__rpow__ -__rrshift__ -__rshift__ -__rsub__ -__rtruediv__ -__rxor__ -__setattr__ -__setitem__ -__str__ -__sub__ -__truediv__ -__xor__ - -""".strip().split()) - - -def eval_ast_expr( - node: ast.expr, - *, - filename: str = '-' -) -> Any: - """ - Takes an ast.Expr node. Compiles it into a function object, - then calls the function object with 0 arguments. - Returns the result of that function call. - - globals represents the globals dict the expression - should see. (There's no equivalent for "locals" here.) - """ - - if isinstance(node, ast.Expr): - node = node.value - - expr = ast.Expression(node) - namespace = create_parser_namespace() - co = compile(expr, filename, 'eval') - fn = FunctionType(co, namespace) - return fn() - - -class IndentStack: - def __init__(self) -> None: - self.indents: list[int] = [] - self.margin: str | None = None - - def _ensure(self) -> None: - if not self.indents: - fail('IndentStack expected indents, but none are defined.') - - def measure(self, line: str) -> int: - """ - Returns the length of the line's margin. - """ - if '\t' in line: - fail('Tab characters are illegal in the Argument Clinic DSL.') - stripped = line.lstrip() - if not len(stripped): - # we can't tell anything from an empty line - # so just pretend it's indented like our current indent - self._ensure() - return self.indents[-1] - return len(line) - len(stripped) - - def infer(self, line: str) -> int: - """ - Infer what is now the current margin based on this line. - Returns: - 1 if we have indented (or this is the first margin) - 0 if the margin has not changed - -N if we have dedented N times - """ - indent = self.measure(line) - margin = ' ' * indent - if not self.indents: - self.indents.append(indent) - self.margin = margin - return 1 - current = self.indents[-1] - if indent == current: - return 0 - if indent > current: - self.indents.append(indent) - self.margin = margin - return 1 - # indent < current - if indent not in self.indents: - fail("Illegal outdent.") - outdent_count = 0 - while indent != current: - self.indents.pop() - current = self.indents[-1] - outdent_count -= 1 - self.margin = margin - return outdent_count - - @property - def depth(self) -> int: - """ - Returns how many margins are currently defined. - """ - return len(self.indents) - - def dedent(self, line: str) -> str: - """ - Dedents a line by the currently defined margin. - """ - assert self.margin is not None, "Cannot call .dedent() before calling .infer()" - margin = self.margin - indent = self.indents[-1] - if not line.startswith(margin): - fail('Cannot dedent; line does not start with the previous margin.') - return line[indent:] - - -StateKeeper = Callable[[str], None] -ConverterArgs = dict[str, Any] - -class ParamState(enum.IntEnum): - """Parameter parsing state. - - [ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] <- line - 01 2 3 4 5 6 <- state transitions - """ - # Before we've seen anything. - # Legal transitions: to LEFT_SQUARE_BEFORE or REQUIRED - START = 0 - - # Left square backets before required params. - LEFT_SQUARE_BEFORE = 1 - - # In a group, before required params. - GROUP_BEFORE = 2 - - # Required params, positional-or-keyword or positional-only (we - # don't know yet). Renumber left groups! - REQUIRED = 3 - - # Positional-or-keyword or positional-only params that now must have - # default values. - OPTIONAL = 4 - - # In a group, after required params. - GROUP_AFTER = 5 - - # Right square brackets after required params. - RIGHT_SQUARE_AFTER = 6 - - -class FunctionNames(NamedTuple): - full_name: str - c_basename: str - - -class DSLParser: - function: Function | None - state: StateKeeper - keyword_only: bool - positional_only: bool - deprecated_positional: VersionTuple | None - deprecated_keyword: VersionTuple | None - group: int - parameter_state: ParamState - indent: IndentStack - kind: FunctionKind - coexist: bool - forced_text_signature: str | None - parameter_continuation: str - preserve_output: bool - critical_section: bool - target_critical_section: list[str] - from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') - - def __init__(self, clinic: Clinic) -> None: - self.clinic = clinic - - self.directives = {} - for name in dir(self): - # functions that start with directive_ are added to directives - _, s, key = name.partition("directive_") - if s: - self.directives[key] = getattr(self, name) - - # functions that start with at_ are too, with an @ in front - _, s, key = name.partition("at_") - if s: - self.directives['@' + key] = getattr(self, name) - - self.reset() - - def reset(self) -> None: - self.function = None - self.state = self.state_dsl_start - self.keyword_only = False - self.positional_only = False - self.deprecated_positional = None - self.deprecated_keyword = None - self.group = 0 - self.parameter_state: ParamState = ParamState.START - self.indent = IndentStack() - self.kind = CALLABLE - self.coexist = False - self.forced_text_signature = None - self.parameter_continuation = '' - self.preserve_output = False - self.critical_section = False - self.target_critical_section = [] - - def directive_module(self, name: str) -> None: - fields = name.split('.')[:-1] - module, cls = self.clinic._module_and_class(fields) - if cls: - fail("Can't nest a module inside a class!") - - if name in module.modules: - fail(f"Already defined module {name!r}!") - - m = Module(name, module) - module.modules[name] = m - self.block.signatures.append(m) - - def directive_class( - self, - name: str, - typedef: str, - type_object: str - ) -> None: - fields = name.split('.') - name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - parent = cls or module - if name in parent.classes: - fail(f"Already defined class {name!r}!") - - c = Class(name, module, cls, typedef, type_object) - parent.classes[name] = c - self.block.signatures.append(c) - - def directive_set(self, name: str, value: str) -> None: - if name not in ("line_prefix", "line_suffix"): - fail(f"unknown variable {name!r}") - - value = value.format_map({ - 'block comment start': '/*', - 'block comment end': '*/', - }) - - self.clinic.__dict__[name] = value - - def directive_destination( - self, - name: str, - command: str, - *args: str - ) -> None: - match command: - case "new": - self.clinic.add_destination(name, *args) - case "clear": - self.clinic.get_destination(name).clear() - case _: - fail(f"unknown destination command {command!r}") - - - def directive_output( - self, - command_or_name: str, - destination: str = '' - ) -> None: - fd = self.clinic.destination_buffers - - if command_or_name == "preset": - preset = self.clinic.presets.get(destination) - if not preset: - fail(f"Unknown preset {destination!r}!") - fd.update(preset) - return - - if command_or_name == "push": - self.clinic.destination_buffers_stack.append(fd.copy()) - return - - if command_or_name == "pop": - if not self.clinic.destination_buffers_stack: - fail("Can't 'output pop', stack is empty!") - previous_fd = self.clinic.destination_buffers_stack.pop() - fd.update(previous_fd) - return - - # secret command for debugging! - if command_or_name == "print": - self.block.output.append(pprint.pformat(fd)) - self.block.output.append('\n') - return - - d = self.clinic.get_destination_buffer(destination) - - if command_or_name == "everything": - for name in list(fd): - fd[name] = d - return - - if command_or_name not in fd: - allowed = ["preset", "push", "pop", "print", "everything"] - allowed.extend(fd) - fail(f"Invalid command or destination name {command_or_name!r}. " - "Must be one of:\n -", - "\n - ".join([repr(word) for word in allowed])) - fd[command_or_name] = d - - def directive_dump(self, name: str) -> None: - self.block.output.append(self.clinic.get_destination(name).dump()) - - def directive_printout(self, *args: str) -> None: - self.block.output.append(' '.join(args)) - self.block.output.append('\n') - - def directive_preserve(self) -> None: - if self.preserve_output: - fail("Can't have 'preserve' twice in one block!") - self.preserve_output = True - - def at_classmethod(self) -> None: - if self.kind is not CALLABLE: - fail("Can't set @classmethod, function is not a normal callable") - self.kind = CLASS_METHOD - - def at_critical_section(self, *args: str) -> None: - if len(args) > 2: - fail("Up to 2 critical section variables are supported") - self.target_critical_section.extend(args) - self.critical_section = True - - def at_getter(self) -> None: - match self.kind: - case FunctionKind.GETTER: - fail("Cannot apply @getter twice to the same function!") - case FunctionKind.SETTER: - fail("Cannot apply both @getter and @setter to the same function!") - case _: - self.kind = FunctionKind.GETTER - - def at_setter(self) -> None: - match self.kind: - case FunctionKind.SETTER: - fail("Cannot apply @setter twice to the same function!") - case FunctionKind.GETTER: - fail("Cannot apply both @getter and @setter to the same function!") - case _: - self.kind = FunctionKind.SETTER - - def at_staticmethod(self) -> None: - if self.kind is not CALLABLE: - fail("Can't set @staticmethod, function is not a normal callable") - self.kind = STATIC_METHOD - - def at_coexist(self) -> None: - if self.coexist: - fail("Called @coexist twice!") - self.coexist = True - - def at_text_signature(self, text_signature: str) -> None: - if self.forced_text_signature: - fail("Called @text_signature twice!") - self.forced_text_signature = text_signature - - def parse(self, block: Block) -> None: - self.reset() - self.block = block - self.saved_output = self.block.output - block.output = [] - block_start = self.clinic.block_parser.line_number - lines = block.input.split('\n') - for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): - if '\t' in line: - fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', - line_number=block_start) - try: - self.state(line) - except ClinicError as exc: - exc.lineno = line_number - exc.filename = self.clinic.filename - raise - - self.do_post_block_processing_cleanup(line_number) - block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) - - if self.preserve_output: - if block.output: - fail("'preserve' only works for blocks that don't produce any output!", - line_number=line_number) - block.output = self.saved_output - - def in_docstring(self) -> bool: - """Return true if we are processing a docstring.""" - return self.state in { - self.state_parameter_docstring, - self.state_function_docstring, - } - - def valid_line(self, line: str) -> bool: - # ignore comment-only lines - if line.lstrip().startswith('#'): - return False - - # Ignore empty lines too - # (but not in docstring sections!) - if not self.in_docstring() and not line.strip(): - return False - - return True - - def next( - self, - state: StateKeeper, - line: str | None = None - ) -> None: - self.state = state - if line is not None: - self.state(line) - - def state_dsl_start(self, line: str) -> None: - if not self.valid_line(line): - return - - # is it a directive? - fields = shlex.split(line) - directive_name = fields[0] - directive = self.directives.get(directive_name, None) - if directive: - try: - directive(*fields[1:]) - except TypeError as e: - fail(str(e)) - return - - self.next(self.state_modulename_name, line) - - def parse_function_names(self, line: str) -> FunctionNames: - left, as_, right = line.partition(' as ') - full_name = left.strip() - c_basename = right.strip() - if as_ and not c_basename: - fail("No C basename provided after 'as' keyword") - if not c_basename: - fields = full_name.split(".") - if fields[-1] == '__new__': - fields.pop() - c_basename = "_".join(fields) - if not libclinic.is_legal_py_identifier(full_name): - fail(f"Illegal function name: {full_name!r}") - if not libclinic.is_legal_c_identifier(c_basename): - fail(f"Illegal C basename: {c_basename!r}") - names = FunctionNames(full_name=full_name, c_basename=c_basename) - self.normalize_function_kind(names.full_name) - return names - - def normalize_function_kind(self, fullname: str) -> None: - # Fetch the method name and possibly class. - fields = fullname.split('.') - name = fields.pop() - _, cls = self.clinic._module_and_class(fields) - - # Check special method requirements. - if name in unsupported_special_methods: - fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") - if name == '__init__' and (self.kind is not CALLABLE or not cls): - fail(f"{name!r} must be a normal method; got '{self.kind}'!") - if name == '__new__' and (self.kind is not CLASS_METHOD or not cls): - fail("'__new__' must be a class method!") - if self.kind in {GETTER, SETTER} and not cls: - fail("@getter and @setter must be methods") - - # Normalise self.kind. - if name == '__new__': - self.kind = METHOD_NEW - elif name == '__init__': - self.kind = METHOD_INIT - - def resolve_return_converter( - self, full_name: str, forced_converter: str - ) -> CReturnConverter: - if forced_converter: - if self.kind in {GETTER, SETTER}: - fail(f"@{self.kind.name.lower()} method cannot define a return type") - if self.kind is METHOD_INIT: - fail("__init__ methods cannot define a return type") - ast_input = f"def x() -> {forced_converter}: pass" - try: - module_node = ast.parse(ast_input) - except SyntaxError: - fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") - function_node = module_node.body[0] - assert isinstance(function_node, ast.FunctionDef) - try: - name, legacy, kwargs = self.parse_converter(function_node.returns) - if legacy: - fail(f"Legacy converter {name!r} not allowed as a return converter") - if name not in return_converters: - fail(f"No available return converter called {name!r}") - return return_converters[name](**kwargs) - except ValueError: - fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") - - if self.kind in {METHOD_INIT, SETTER}: - return int_return_converter() - return CReturnConverter() - - def parse_cloned_function(self, names: FunctionNames, existing: str) -> None: - full_name, c_basename = names - fields = [x.strip() for x in existing.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - parent = cls or module - - for existing_function in parent.functions: - if existing_function.name == function_name: - break - else: - print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) - print(f"{(cls or module).functions=}", file=sys.stderr) - fail(f"Couldn't find existing function {existing!r}!") - - fields = [x.strip() for x in full_name.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - overrides: dict[str, Any] = { - "name": function_name, - "full_name": full_name, - "module": module, - "cls": cls, - "c_basename": c_basename, - "docstring": "", - } - if not (existing_function.kind is self.kind and - existing_function.coexist == self.coexist): - # Allow __new__ or __init__ methods. - if existing_function.kind.new_or_init: - overrides["kind"] = self.kind - # Future enhancement: allow custom return converters - overrides["return_converter"] = CReturnConverter() - else: - fail("'kind' of function and cloned function don't match! " - "(@classmethod/@staticmethod/@coexist)") - function = existing_function.copy(**overrides) - self.function = function - self.block.signatures.append(function) - (cls or module).functions.append(function) - self.next(self.state_function_docstring) - - def state_modulename_name(self, line: str) -> None: - # looking for declaration, which establishes the leftmost column - # line should be - # modulename.fnname [as c_basename] [-> return annotation] - # square brackets denote optional syntax. - # - # alternatively: - # modulename.fnname [as c_basename] = modulename.existing_fn_name - # clones the parameters and return converter from that - # function. you can't modify them. you must enter a - # new docstring. - # - # (but we might find a directive first!) - # - # this line is permitted to start with whitespace. - # we'll call this number of spaces F (for "function"). - - assert self.valid_line(line) - self.indent.infer(line) - - # are we cloning? - before, equals, existing = line.rpartition('=') - if equals: - existing = existing.strip() - if libclinic.is_legal_py_identifier(existing): - # we're cloning! - names = self.parse_function_names(before) - return self.parse_cloned_function(names, existing) - - line, _, returns = line.partition('->') - returns = returns.strip() - full_name, c_basename = self.parse_function_names(line) - return_converter = self.resolve_return_converter(full_name, returns) - - fields = [x.strip() for x in full_name.split('.')] - function_name = fields.pop() - module, cls = self.clinic._module_and_class(fields) - - func = Function( - name=function_name, - full_name=full_name, - module=module, - cls=cls, - c_basename=c_basename, - return_converter=return_converter, - kind=self.kind, - coexist=self.coexist, - critical_section=self.critical_section, - target_critical_section=self.target_critical_section - ) - self.add_function(func) - - self.next(self.state_parameters_start) - - def add_function(self, func: Function) -> None: - # Insert a self converter automatically. - tp, name = correct_name_for_self(func) - if func.cls and tp == "PyObject *": - func.self_converter = self_converter(name, name, func, - type=func.cls.typedef) - else: - func.self_converter = self_converter(name, name, func) - func.parameters[name] = Parameter( - name, - inspect.Parameter.POSITIONAL_ONLY, - function=func, - converter=func.self_converter - ) - - self.block.signatures.append(func) - self.function = func - (func.cls or func.module).functions.append(func) - - # Now entering the parameters section. The rules, formally stated: - # - # * All lines must be indented with spaces only. - # * The first line must be a parameter declaration. - # * The first line must be indented. - # * This first line establishes the indent for parameters. - # * We'll call this number of spaces P (for "parameter"). - # * Thenceforth: - # * Lines indented with P spaces specify a parameter. - # * Lines indented with > P spaces are docstrings for the previous - # parameter. - # * We'll call this number of spaces D (for "docstring"). - # * All subsequent lines indented with >= D spaces are stored as - # part of the per-parameter docstring. - # * All lines will have the first D spaces of the indent stripped - # before they are stored. - # * It's illegal to have a line starting with a number of spaces X - # such that P < X < D. - # * A line with < P spaces is the first line of the function - # docstring, which ends processing for parameters and per-parameter - # docstrings. - # * The first line of the function docstring must be at the same - # indent as the function declaration. - # * It's illegal to have any line in the parameters section starting - # with X spaces such that F < X < P. (As before, F is the indent - # of the function declaration.) - # - # Also, currently Argument Clinic places the following restrictions on groups: - # * Each group must contain at least one parameter. - # * Each group may contain at most one group, which must be the furthest - # thing in the group from the required parameters. (The nested group - # must be the first in the group when it's before the required - # parameters, and the last thing in the group when after the required - # parameters.) - # * There may be at most one (top-level) group to the left or right of - # the required parameters. - # * You must specify a slash, and it must be after all parameters. - # (In other words: either all parameters are positional-only, - # or none are.) - # - # Said another way: - # * Each group must contain at least one parameter. - # * All left square brackets before the required parameters must be - # consecutive. (You can't have a left square bracket followed - # by a parameter, then another left square bracket. You can't - # have a left square bracket, a parameter, a right square bracket, - # and then a left square bracket.) - # * All right square brackets after the required parameters must be - # consecutive. - # - # These rules are enforced with a single state variable: - # "parameter_state". (Previously the code was a miasma of ifs and - # separate boolean state variables.) The states are defined in the - # ParamState class. - - def state_parameters_start(self, line: str) -> None: - if not self.valid_line(line): - return - - # if this line is not indented, we have no parameters - if not self.indent.infer(line): - return self.next(self.state_function_docstring, line) - - assert self.function is not None - if self.function.kind in {GETTER, SETTER}: - getset = self.function.kind.name.lower() - fail(f"@{getset} methods cannot define parameters") - - self.parameter_continuation = '' - return self.next(self.state_parameter, line) - - - def to_required(self) -> None: - """ - Transition to the "required" parameter state. - """ - if self.parameter_state is not ParamState.REQUIRED: - self.parameter_state = ParamState.REQUIRED - assert self.function is not None - for p in self.function.parameters.values(): - p.group = -p.group - - def state_parameter(self, line: str) -> None: - assert isinstance(self.function, Function) - - if not self.valid_line(line): - return - - if self.parameter_continuation: - line = self.parameter_continuation + ' ' + line.lstrip() - self.parameter_continuation = '' - - assert self.indent.depth == 2 - indent = self.indent.infer(line) - if indent == -1: - # we outdented, must be to definition column - return self.next(self.state_function_docstring, line) - - if indent == 1: - # we indented, must be to new parameter docstring column - return self.next(self.state_parameter_docstring_start, line) - - line = line.rstrip() - if line.endswith('\\'): - self.parameter_continuation = line[:-1] - return - - line = line.lstrip() - version: VersionTuple | None = None - match = self.from_version_re.fullmatch(line) - if match: - line = match[1] - version = self.parse_version(match[2]) - - func = self.function - match line: - case '*': - self.parse_star(func, version) - case '[': - self.parse_opening_square_bracket(func) - case ']': - self.parse_closing_square_bracket(func) - case '/': - self.parse_slash(func, version) - case param: - self.parse_parameter(param) - - def parse_parameter(self, line: str) -> None: - assert self.function is not None - - match self.parameter_state: - case ParamState.START | ParamState.REQUIRED: - self.to_required() - case ParamState.LEFT_SQUARE_BEFORE: - self.parameter_state = ParamState.GROUP_BEFORE - case ParamState.GROUP_BEFORE: - if not self.group: - self.to_required() - case ParamState.GROUP_AFTER | ParamState.OPTIONAL: - pass - case st: - fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.a)") - - # handle "as" for parameters too - c_name = None - name, have_as_token, trailing = line.partition(' as ') - if have_as_token: - name = name.strip() - if ' ' not in name: - fields = trailing.strip().split(' ') - if not fields: - fail("Invalid 'as' clause!") - c_name = fields[0] - if c_name.endswith(':'): - name += ':' - c_name = c_name[:-1] - fields[0] = name - line = ' '.join(fields) - - default: str | None - base, equals, default = line.rpartition('=') - if not equals: - base = default - default = None - - module = None - try: - ast_input = f"def x({base}): pass" - module = ast.parse(ast_input) - except SyntaxError: - try: - # the last = was probably inside a function call, like - # c: int(accept={str}) - # so assume there was no actual default value. - default = None - ast_input = f"def x({line}): pass" - module = ast.parse(ast_input) - except SyntaxError: - pass - if not module: - fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", - repr(line)) - - function = module.body[0] - assert isinstance(function, ast.FunctionDef) - function_args = function.args - - if len(function_args.args) > 1: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (comma?): {line!r}") - if function_args.defaults or function_args.kw_defaults: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (default value?): {line!r}") - if function_args.kwarg: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (**kwargs?): {line!r}") - - if function_args.vararg: - if any(p.is_vararg() for p in self.function.parameters.values()): - fail("Too many var args") - is_vararg = True - parameter = function_args.vararg - else: - is_vararg = False - parameter = function_args.args[0] - - parameter_name = parameter.arg - name, legacy, kwargs = self.parse_converter(parameter.annotation) - - value: object - if not default: - if self.parameter_state is ParamState.OPTIONAL: - fail(f"Can't have a parameter without a default ({parameter_name!r}) " - "after a parameter with a default!") - if is_vararg: - value = NULL - kwargs.setdefault('c_default', "NULL") - else: - value = unspecified - if 'py_default' in kwargs: - fail("You can't specify py_default without specifying a default value!") - else: - if is_vararg: - fail("Vararg can't take a default value!") - - if self.parameter_state is ParamState.REQUIRED: - self.parameter_state = ParamState.OPTIONAL - default = default.strip() - bad = False - ast_input = f"x = {default}" - try: - module = ast.parse(ast_input) - - if 'c_default' not in kwargs: - # we can only represent very simple data values in C. - # detect whether default is okay, via a denylist - # of disallowed ast nodes. - class DetectBadNodes(ast.NodeVisitor): - bad = False - def bad_node(self, node: ast.AST) -> None: - self.bad = True - - # inline function call - visit_Call = bad_node - # inline if statement ("x = 3 if y else z") - visit_IfExp = bad_node - - # comprehensions and generator expressions - visit_ListComp = visit_SetComp = bad_node - visit_DictComp = visit_GeneratorExp = bad_node - - # literals for advanced types - visit_Dict = visit_Set = bad_node - visit_List = visit_Tuple = bad_node - - # "starred": "a = [1, 2, 3]; *a" - visit_Starred = bad_node - - denylist = DetectBadNodes() - denylist.visit(module) - bad = denylist.bad - else: - # if they specify a c_default, we can be more lenient about the default value. - # but at least make an attempt at ensuring it's a valid expression. - try: - value = eval(default) - except NameError: - pass # probably a named constant - except Exception as e: - fail("Malformed expression given as default value " - f"{default!r} caused {e!r}") - else: - if value is unspecified: - fail("'unspecified' is not a legal default value!") - if bad: - fail(f"Unsupported expression as default value: {default!r}") - - assignment = module.body[0] - assert isinstance(assignment, ast.Assign) - expr = assignment.value - # mild hack: explicitly support NULL as a default value - c_default: str | None - if isinstance(expr, ast.Name) and expr.id == 'NULL': - value = NULL - py_default = '' - c_default = "NULL" - elif (isinstance(expr, ast.BinOp) or - (isinstance(expr, ast.UnaryOp) and - not (isinstance(expr.operand, ast.Constant) and - type(expr.operand.value) in {int, float, complex}) - )): - c_default = kwargs.get("c_default") - if not (isinstance(c_default, str) and c_default): - fail(f"When you specify an expression ({default!r}) " - f"as your default value, " - f"you MUST specify a valid c_default.", - ast.dump(expr)) - py_default = default - value = unknown - elif isinstance(expr, ast.Attribute): - a = [] - n: ast.expr | ast.Attribute = expr - while isinstance(n, ast.Attribute): - a.append(n.attr) - n = n.value - if not isinstance(n, ast.Name): - fail(f"Unsupported default value {default!r} " - "(looked like a Python constant)") - a.append(n.id) - py_default = ".".join(reversed(a)) - - c_default = kwargs.get("c_default") - if not (isinstance(c_default, str) and c_default): - fail(f"When you specify a named constant ({py_default!r}) " - "as your default value, " - "you MUST specify a valid c_default.") - - try: - value = eval(py_default) - except NameError: - value = unknown - else: - value = ast.literal_eval(expr) - py_default = repr(value) - if isinstance(value, (bool, NoneType)): - c_default = "Py_" + py_default - elif isinstance(value, str): - c_default = libclinic.c_repr(value) - else: - c_default = py_default - - except SyntaxError as e: - fail(f"Syntax error: {e.text!r}") - except (ValueError, AttributeError): - value = unknown - c_default = kwargs.get("c_default") - py_default = default - if not (isinstance(c_default, str) and c_default): - fail("When you specify a named constant " - f"({py_default!r}) as your default value, " - "you MUST specify a valid c_default.") - - kwargs.setdefault('c_default', c_default) - kwargs.setdefault('py_default', py_default) - - dict = legacy_converters if legacy else converters - legacy_str = "legacy " if legacy else "" - if name not in dict: - fail(f'{name!r} is not a valid {legacy_str}converter') - # if you use a c_name for the parameter, we just give that name to the converter - # but the parameter object gets the python name - converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) - - kind: inspect._ParameterKind - if is_vararg: - kind = inspect.Parameter.VAR_POSITIONAL - elif self.keyword_only: - kind = inspect.Parameter.KEYWORD_ONLY - else: - kind = inspect.Parameter.POSITIONAL_OR_KEYWORD - - if isinstance(converter, self_converter): - if len(self.function.parameters) == 1: - if self.parameter_state is not ParamState.REQUIRED: - fail("A 'self' parameter cannot be marked optional.") - if value is not unspecified: - fail("A 'self' parameter cannot have a default value.") - if self.group: - fail("A 'self' parameter cannot be in an optional group.") - kind = inspect.Parameter.POSITIONAL_ONLY - self.parameter_state = ParamState.START - self.function.parameters.clear() - else: - fail("A 'self' parameter, if specified, must be the " - "very first thing in the parameter block.") - - if isinstance(converter, defining_class_converter): - _lp = len(self.function.parameters) - if _lp == 1: - if self.parameter_state is not ParamState.REQUIRED: - fail("A 'defining_class' parameter cannot be marked optional.") - if value is not unspecified: - fail("A 'defining_class' parameter cannot have a default value.") - if self.group: - fail("A 'defining_class' parameter cannot be in an optional group.") - else: - fail("A 'defining_class' parameter, if specified, must either " - "be the first thing in the parameter block, or come just " - "after 'self'.") - - - p = Parameter(parameter_name, kind, function=self.function, - converter=converter, default=value, group=self.group, - deprecated_positional=self.deprecated_positional) - - names = [k.name for k in self.function.parameters.values()] - if parameter_name in names[1:]: - fail(f"You can't have two parameters named {parameter_name!r}!") - elif names and parameter_name == names[0] and c_name is None: - fail(f"Parameter {parameter_name!r} requires a custom C name") - - key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name - self.function.parameters[key] = p - - @staticmethod - def parse_converter( - annotation: ast.expr | None - ) -> tuple[str, bool, ConverterArgs]: - match annotation: - case ast.Constant(value=str() as value): - return value, True, {} - case ast.Name(name): - return name, False, {} - case ast.Call(func=ast.Name(name)): - kwargs: ConverterArgs = {} - for node in annotation.keywords: - if not isinstance(node.arg, str): - fail("Cannot use a kwarg splat in a function-call annotation") - kwargs[node.arg] = eval_ast_expr(node.value) - return name, False, kwargs - case _: - fail( - "Annotations must be either a name, a function call, or a string." - ) - - def parse_version(self, thenceforth: str) -> VersionTuple: - """Parse Python version in `[from ...]` marker.""" - assert isinstance(self.function, Function) - - try: - major, minor = thenceforth.split(".") - return int(major), int(minor) - except ValueError: - fail( - f"Function {self.function.name!r}: expected format '[from major.minor]' " - f"where 'major' and 'minor' are integers; got {thenceforth!r}" - ) - - def parse_star(self, function: Function, version: VersionTuple | None) -> None: - """Parse keyword-only parameter marker '*'. - - The 'version' parameter signifies the future version from which - the marker will take effect (None means it is already in effect). - """ - if version is None: - if self.keyword_only: - fail(f"Function {function.name!r} uses '*' more than once.") - self.check_previous_star() - self.check_remaining_star() - self.keyword_only = True - else: - if self.keyword_only: - fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") - if self.deprecated_positional: - if self.deprecated_positional == version: - fail(f"Function {function.name!r} uses '* [from " - f"{version[0]}.{version[1]}]' more than once.") - if self.deprecated_positional < version: - fail(f"Function {function.name!r}: '* [from " - f"{version[0]}.{version[1]}]' must precede '* [from " - f"{self.deprecated_positional[0]}.{self.deprecated_positional[1]}]'") - self.deprecated_positional = version - - def parse_opening_square_bracket(self, function: Function) -> None: - """Parse opening parameter group symbol '['.""" - match self.parameter_state: - case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: - self.parameter_state = ParamState.LEFT_SQUARE_BEFORE - case ParamState.REQUIRED | ParamState.GROUP_AFTER: - self.parameter_state = ParamState.GROUP_AFTER - case st: - fail(f"Function {function.name!r} " - f"has an unsupported group configuration. " - f"(Unexpected state {st}.b)") - self.group += 1 - function.docstring_only = True - - def parse_closing_square_bracket(self, function: Function) -> None: - """Parse closing parameter group symbol ']'.""" - if not self.group: - fail(f"Function {function.name!r} has a ']' without a matching '['.") - if not any(p.group == self.group for p in function.parameters.values()): - fail(f"Function {function.name!r} has an empty group. " - "All groups must contain at least one parameter.") - self.group -= 1 - match self.parameter_state: - case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: - self.parameter_state = ParamState.GROUP_BEFORE - case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: - self.parameter_state = ParamState.RIGHT_SQUARE_AFTER - case st: - fail(f"Function {function.name!r} " - f"has an unsupported group configuration. " - f"(Unexpected state {st}.c)") - - def parse_slash(self, function: Function, version: VersionTuple | None) -> None: - """Parse positional-only parameter marker '/'. - - The 'version' parameter signifies the future version from which - the marker will take effect (None means it is already in effect). - """ - if version is None: - if self.deprecated_keyword: - fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") - if self.deprecated_positional: - fail(f"Function {function.name!r}: '/' must precede '* [from ...]'") - if self.keyword_only: - fail(f"Function {function.name!r}: '/' must precede '*'") - if self.positional_only: - fail(f"Function {function.name!r} uses '/' more than once.") - else: - if self.deprecated_keyword: - if self.deprecated_keyword == version: - fail(f"Function {function.name!r} uses '/ [from " - f"{version[0]}.{version[1]}]' more than once.") - if self.deprecated_keyword > version: - fail(f"Function {function.name!r}: '/ [from " - f"{version[0]}.{version[1]}]' must precede '/ [from " - f"{self.deprecated_keyword[0]}.{self.deprecated_keyword[1]}]'") - if self.deprecated_positional: - fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") - if self.keyword_only: - fail(f"Function {function.name!r}: '/ [from ...]' must precede '*'") - self.positional_only = True - self.deprecated_keyword = version - if version is not None: - found = False - for p in reversed(function.parameters.values()): - found = p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD - break - if not found: - fail(f"Function {function.name!r} specifies '/ [from ...]' " - f"without preceding parameters.") - # REQUIRED and OPTIONAL are allowed here, that allows positional-only - # without option groups to work (and have default values!) - allowed = { - ParamState.REQUIRED, - ParamState.OPTIONAL, - ParamState.RIGHT_SQUARE_AFTER, - ParamState.GROUP_BEFORE, - } - if (self.parameter_state not in allowed) or self.group: - fail(f"Function {function.name!r} has an unsupported group configuration. " - f"(Unexpected state {self.parameter_state}.d)") - # fixup preceding parameters - for p in function.parameters.values(): - if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: - if version is None: - p.kind = inspect.Parameter.POSITIONAL_ONLY - elif p.deprecated_keyword is None: - p.deprecated_keyword = version - - def state_parameter_docstring_start(self, line: str) -> None: - assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" - self.parameter_docstring_indent = len(self.indent.margin) - assert self.indent.depth == 3 - return self.next(self.state_parameter_docstring, line) - - def docstring_append(self, obj: Function | Parameter, line: str) -> None: - """Add a rstripped line to the current docstring.""" - # gh-80282: We filter out non-ASCII characters from the docstring, - # since historically, some compilers may balk on non-ASCII input. - # If you're using Argument Clinic in an external project, - # you may not need to support the same array of platforms as CPython, - # so you may be able to remove this restriction. - matches = re.finditer(r'[^\x00-\x7F]', line) - if offending := ", ".join([repr(m[0]) for m in matches]): - warn("Non-ascii characters are not allowed in docstrings:", - offending) - - docstring = obj.docstring - if docstring: - docstring += "\n" - if stripped := line.rstrip(): - docstring += self.indent.dedent(stripped) - obj.docstring = docstring - - # every line of the docstring must start with at least F spaces, - # where F > P. - # these F spaces will be stripped. - def state_parameter_docstring(self, line: str) -> None: - if not self.valid_line(line): - return - - indent = self.indent.measure(line) - if indent < self.parameter_docstring_indent: - self.indent.infer(line) - assert self.indent.depth < 3 - if self.indent.depth == 2: - # back to a parameter - return self.next(self.state_parameter, line) - assert self.indent.depth == 1 - return self.next(self.state_function_docstring, line) - - assert self.function and self.function.parameters - last_param = next(reversed(self.function.parameters.values())) - self.docstring_append(last_param, line) - - # the final stanza of the DSL is the docstring. - def state_function_docstring(self, line: str) -> None: - assert self.function is not None - - if self.group: - fail(f"Function {self.function.name!r} has a ']' without a matching '['.") - - if not self.valid_line(line): - return - - self.docstring_append(self.function, line) - - def format_docstring_signature( - self, f: Function, parameters: list[Parameter] - ) -> str: - lines = [] - lines.append(f.displayname) - if self.forced_text_signature: - lines.append(self.forced_text_signature) - elif f.kind in {GETTER, SETTER}: - # @getter and @setter do not need signatures like a method or a function. - return '' - else: - lines.append('(') - - # populate "right_bracket_count" field for every parameter - assert parameters, "We should always have a self parameter. " + repr(f) - assert isinstance(parameters[0].converter, self_converter) - # self is always positional-only. - assert parameters[0].is_positional_only() - assert parameters[0].right_bracket_count == 0 - positional_only = True - for p in parameters[1:]: - if not p.is_positional_only(): - positional_only = False - else: - assert positional_only - if positional_only: - p.right_bracket_count = abs(p.group) - else: - # don't put any right brackets around non-positional-only parameters, ever. - p.right_bracket_count = 0 - - right_bracket_count = 0 - - def fix_right_bracket_count(desired: int) -> str: - nonlocal right_bracket_count - s = '' - while right_bracket_count < desired: - s += '[' - right_bracket_count += 1 - while right_bracket_count > desired: - s += ']' - right_bracket_count -= 1 - return s - - need_slash = False - added_slash = False - need_a_trailing_slash = False - - # we only need a trailing slash: - # * if this is not a "docstring_only" signature - # * and if the last *shown* parameter is - # positional only - if not f.docstring_only: - for p in reversed(parameters): - if not p.converter.show_in_signature: - continue - if p.is_positional_only(): - need_a_trailing_slash = True - break - - - added_star = False - - first_parameter = True - last_p = parameters[-1] - line_length = len(''.join(lines)) - indent = " " * line_length - def add_parameter(text: str) -> None: - nonlocal line_length - nonlocal first_parameter - if first_parameter: - s = text - first_parameter = False - else: - s = ' ' + text - if line_length + len(s) >= 72: - lines.extend(["\n", indent]) - line_length = len(indent) - s = text - line_length += len(s) - lines.append(s) - - for p in parameters: - if not p.converter.show_in_signature: - continue - assert p.name - - is_self = isinstance(p.converter, self_converter) - if is_self and f.docstring_only: - # this isn't a real machine-parsable signature, - # so let's not print the "self" parameter - continue - - if p.is_positional_only(): - need_slash = not f.docstring_only - elif need_slash and not (added_slash or p.is_positional_only()): - added_slash = True - add_parameter('/,') - - if p.is_keyword_only() and not added_star: - added_star = True - add_parameter('*,') - - p_lines = [fix_right_bracket_count(p.right_bracket_count)] - - if isinstance(p.converter, self_converter): - # annotate first parameter as being a "self". - # - # if inspect.Signature gets this function, - # and it's already bound, the self parameter - # will be stripped off. - # - # if it's not bound, it should be marked - # as positional-only. - # - # note: we don't print "self" for __init__, - # because this isn't actually the signature - # for __init__. (it can't be, __init__ doesn't - # have a docstring.) if this is an __init__ - # (or __new__), then this signature is for - # calling the class to construct a new instance. - p_lines.append('$') - - if p.is_vararg(): - p_lines.append("*") - - name = p.converter.signature_name or p.name - p_lines.append(name) - - if not p.is_vararg() and p.converter.is_optional(): - p_lines.append('=') - value = p.converter.py_default - if not value: - value = repr(p.converter.default) - p_lines.append(value) - - if (p != last_p) or need_a_trailing_slash: - p_lines.append(',') - - p_output = "".join(p_lines) - add_parameter(p_output) - - lines.append(fix_right_bracket_count(0)) - if need_a_trailing_slash: - add_parameter('/') - lines.append(')') - - # PEP 8 says: - # - # The Python standard library will not use function annotations - # as that would result in a premature commitment to a particular - # annotation style. Instead, the annotations are left for users - # to discover and experiment with useful annotation styles. - # - # therefore this is commented out: - # - # if f.return_converter.py_default: - # lines.append(' -> ') - # lines.append(f.return_converter.py_default) - - if not f.docstring_only: - lines.append("\n" + libclinic.SIG_END_MARKER + "\n") - - signature_line = "".join(lines) - - # now fix up the places where the brackets look wrong - return signature_line.replace(', ]', ',] ') - - @staticmethod - def format_docstring_parameters(params: list[Parameter]) -> str: - """Create substitution text for {parameters}""" - return "".join(p.render_docstring() + "\n" for p in params if p.docstring) - - def format_docstring(self) -> str: - assert self.function is not None - f = self.function - # For the following special cases, it does not make sense to render a docstring. - if f.kind in {METHOD_INIT, METHOD_NEW, GETTER, SETTER} and not f.docstring: - return f.docstring - - # Enforce the summary line! - # The first line of a docstring should be a summary of the function. - # It should fit on one line (80 columns? 79 maybe?) and be a paragraph - # by itself. - # - # Argument Clinic enforces the following rule: - # * either the docstring is empty, - # * or it must have a summary line. - # - # Guido said Clinic should enforce this: - # http://mail.python.org/pipermail/python-dev/2013-June/127110.html - - lines = f.docstring.split('\n') - if len(lines) >= 2: - if lines[1]: - fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" - "Every non-blank function docstring must start with " - "a single line summary followed by an empty line.") - elif len(lines) == 1: - # the docstring is only one line right now--the summary line. - # add an empty line after the summary line so we have space - # between it and the {parameters} we're about to add. - lines.append('') - - parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 - if parameters_marker_count > 1: - fail('You may not specify {parameters} more than once in a docstring!') - - # insert signature at front and params after the summary line - if not parameters_marker_count: - lines.insert(2, '{parameters}') - lines.insert(0, '{signature}') - - # finalize docstring - params = f.render_parameters - parameters = self.format_docstring_parameters(params) - signature = self.format_docstring_signature(f, params) - docstring = "\n".join(lines) - return libclinic.linear_format(docstring, - signature=signature, - parameters=parameters).rstrip() - - def check_remaining_star(self, lineno: int | None = None) -> None: - assert isinstance(self.function, Function) - - if self.keyword_only: - symbol = '*' - elif self.deprecated_positional: - symbol = '* [from ...]' - else: - return - - for p in reversed(self.function.parameters.values()): - if self.keyword_only: - if p.kind == inspect.Parameter.KEYWORD_ONLY: - return - elif self.deprecated_positional: - if p.deprecated_positional == self.deprecated_positional: - return - break - - fail(f"Function {self.function.name!r} specifies {symbol!r} " - f"without following parameters.", line_number=lineno) - - def check_previous_star(self, lineno: int | None = None) -> None: - assert isinstance(self.function, Function) - - for p in self.function.parameters.values(): - if p.kind == inspect.Parameter.VAR_POSITIONAL: - fail(f"Function {self.function.name!r} uses '*' more than once.") - - - def do_post_block_processing_cleanup(self, lineno: int) -> None: - """ - Called when processing the block is done. - """ - if not self.function: - return - - self.check_remaining_star(lineno) - try: - self.function.docstring = self.format_docstring() - except ClinicError as exc: - exc.lineno = lineno - exc.filename = self.clinic.filename - raise - - - - -# maps strings to callables. -# the callable should return an object -# that implements the clinic parser -# interface (__init__ and parse). -# -# example parsers: -# "clinic", handles the Clinic DSL -# "python", handles running Python code -# -parsers: dict[str, Callable[[Clinic], Parser]] = { - 'clinic': DSLParser, - 'python': PythonParser, -} - - -def create_cli() -> argparse.ArgumentParser: - cmdline = argparse.ArgumentParser( - prog="clinic.py", - description="""Preprocessor for CPython C files. - -The purpose of the Argument Clinic is automating all the boilerplate involved -with writing argument parsing code for builtins and providing introspection -signatures ("docstrings") for CPython builtins. - -For more information see https://devguide.python.org/development-tools/clinic/""") - cmdline.add_argument("-f", "--force", action='store_true', - help="force output regeneration") - cmdline.add_argument("-o", "--output", type=str, - help="redirect file output to OUTPUT") - cmdline.add_argument("-v", "--verbose", action='store_true', - help="enable verbose mode") - cmdline.add_argument("--converters", action='store_true', - help=("print a list of all supported converters " - "and return converters")) - cmdline.add_argument("--make", action='store_true', - help="walk --srcdir to run over all relevant files") - cmdline.add_argument("--srcdir", type=str, default=os.curdir, - help="the directory tree to walk in --make mode") - cmdline.add_argument("--exclude", type=str, action="append", - help=("a file to exclude in --make mode; " - "can be given multiple times")) - cmdline.add_argument("--limited", dest="limited_capi", action='store_true', - help="use the Limited C API") - cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", - help="the list of files to process") - return cmdline - - -def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: - if ns.converters: - if ns.filename: - parser.error( - "can't specify --converters and a filename at the same time" - ) - AnyConverterType = ConverterType | ReturnConverterType - converter_list: list[tuple[str, AnyConverterType]] = [] - return_converter_list: list[tuple[str, AnyConverterType]] = [] - - for name, converter in converters.items(): - converter_list.append(( - name, - converter, - )) - for name, return_converter in return_converters.items(): - return_converter_list.append(( - name, - return_converter - )) - - print() - - print("Legacy converters:") - legacy = sorted(legacy_converters) - print(' ' + ' '.join(c for c in legacy if c[0].isupper())) - print(' ' + ' '.join(c for c in legacy if c[0].islower())) - print() - - for title, attribute, ids in ( - ("Converters", 'converter_init', converter_list), - ("Return converters", 'return_converter_init', return_converter_list), - ): - print(title + ":") - - ids.sort(key=lambda item: item[0].lower()) - longest = -1 - for name, _ in ids: - longest = max(longest, len(name)) - - for name, cls in ids: - callable = getattr(cls, attribute, None) - if not callable: - continue - signature = inspect.signature(callable) - parameters = [] - for parameter_name, parameter in signature.parameters.items(): - if parameter.kind == inspect.Parameter.KEYWORD_ONLY: - if parameter.default != inspect.Parameter.empty: - s = f'{parameter_name}={parameter.default!r}' - else: - s = parameter_name - parameters.append(s) - print(' {}({})'.format(name, ', '.join(parameters))) - print() - print("All converters also accept (c_default=None, py_default=None, annotation=None).") - print("All return converters also accept (py_default=None).") - return - - if ns.make: - if ns.output or ns.filename: - parser.error("can't use -o or filenames with --make") - if not ns.srcdir: - parser.error("--srcdir must not be empty with --make") - if ns.exclude: - excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude] - excludes = [os.path.normpath(f) for f in excludes] - else: - excludes = [] - for root, dirs, files in os.walk(ns.srcdir): - for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): - if rcs_dir in dirs: - dirs.remove(rcs_dir) - for filename in files: - # handle .c, .cpp and .h files - if not filename.endswith(('.c', '.cpp', '.h')): - continue - path = os.path.join(root, filename) - path = os.path.normpath(path) - if path in excludes: - continue - if ns.verbose: - print(path) - parse_file(path, - verify=not ns.force, limited_capi=ns.limited_capi) - return - - if not ns.filename: - parser.error("no input files") - - if ns.output and len(ns.filename) > 1: - parser.error("can't use -o with multiple filenames") - - for filename in ns.filename: - if ns.verbose: - print(filename) - parse_file(filename, output=ns.output, - verify=not ns.force, limited_capi=ns.limited_capi) - - -def main(argv: list[str] | None = None) -> NoReturn: - parser = create_cli() - args = parser.parse_args(argv) - try: - run_clinic(parser, args) - except ClinicError as exc: - sys.stderr.write(exc.report()) - sys.exit(1) - else: - sys.exit(0) +from libclinic.cli import main if __name__ == "__main__": diff --git a/Tools/clinic/libclinic/app.py b/Tools/clinic/libclinic/app.py new file mode 100644 index 00000000000000..47a897712d053e --- /dev/null +++ b/Tools/clinic/libclinic/app.py @@ -0,0 +1,297 @@ +from __future__ import annotations +import os + +from collections.abc import Callable, Sequence +from typing import Any, TYPE_CHECKING + + +import libclinic +from libclinic import fail, warn +from libclinic.function import Class +from libclinic.block_parser import Block, BlockParser +from libclinic.crenderdata import Include +from libclinic.codegen import BlockPrinter, Destination +from libclinic.parser import Parser, PythonParser +from libclinic.dsl_parser import DSLParser +if TYPE_CHECKING: + from libclinic.clanguage import CLanguage + from libclinic.function import ( + Module, Function, ClassDict, ModuleDict) + from libclinic.codegen import DestinationDict + + +# maps strings to callables. +# the callable should return an object +# that implements the clinic parser +# interface (__init__ and parse). +# +# example parsers: +# "clinic", handles the Clinic DSL +# "python", handles running Python code +# +parsers: dict[str, Callable[[Clinic], Parser]] = { + 'clinic': DSLParser, + 'python': PythonParser, +} + + +class Clinic: + + presets_text = """ +preset block +everything block +methoddef_ifndef buffer 1 +docstring_prototype suppress +parser_prototype suppress +cpp_if suppress +cpp_endif suppress + +preset original +everything block +methoddef_ifndef buffer 1 +docstring_prototype suppress +parser_prototype suppress +cpp_if suppress +cpp_endif suppress + +preset file +everything file +methoddef_ifndef file 1 +docstring_prototype suppress +parser_prototype suppress +impl_definition block + +preset buffer +everything buffer +methoddef_ifndef buffer 1 +impl_definition block +docstring_prototype suppress +impl_prototype suppress +parser_prototype suppress + +preset partial-buffer +everything buffer +methoddef_ifndef buffer 1 +docstring_prototype block +impl_prototype suppress +methoddef_define block +parser_prototype block +impl_definition block + +""" + + def __init__( + self, + language: CLanguage, + printer: BlockPrinter | None = None, + *, + filename: str, + limited_capi: bool, + verify: bool = True, + ) -> None: + # maps strings to Parser objects. + # (instantiated from the "parsers" global.) + self.parsers: dict[str, Parser] = {} + self.language: CLanguage = language + if printer: + fail("Custom printers are broken right now") + self.printer = printer or BlockPrinter(language) + self.verify = verify + self.limited_capi = limited_capi + self.filename = filename + self.modules: ModuleDict = {} + self.classes: ClassDict = {} + self.functions: list[Function] = [] + # dict: include name => Include instance + self.includes: dict[str, Include] = {} + + self.line_prefix = self.line_suffix = '' + + self.destinations: DestinationDict = {} + self.add_destination("block", "buffer") + self.add_destination("suppress", "suppress") + self.add_destination("buffer", "buffer") + if filename: + self.add_destination("file", "file", "{dirname}/clinic/{basename}.h") + + d = self.get_destination_buffer + self.destination_buffers = { + 'cpp_if': d('file'), + 'docstring_prototype': d('suppress'), + 'docstring_definition': d('file'), + 'methoddef_define': d('file'), + 'impl_prototype': d('file'), + 'parser_prototype': d('suppress'), + 'parser_definition': d('file'), + 'cpp_endif': d('file'), + 'methoddef_ifndef': d('file', 1), + 'impl_definition': d('block'), + } + + DestBufferType = dict[str, list[str]] + DestBufferList = list[DestBufferType] + + self.destination_buffers_stack: DestBufferList = [] + self.ifndef_symbols: set[str] = set() + + self.presets: dict[str, dict[Any, Any]] = {} + preset = None + for line in self.presets_text.strip().split('\n'): + line = line.strip() + if not line: + continue + name, value, *options = line.split() + if name == 'preset': + self.presets[value] = preset = {} + continue + + if len(options): + index = int(options[0]) + else: + index = 0 + buffer = self.get_destination_buffer(value, index) + + if name == 'everything': + for name in self.destination_buffers: + preset[name] = buffer + continue + + assert name in self.destination_buffers + preset[name] = buffer + + def add_include(self, name: str, reason: str, + *, condition: str | None = None) -> None: + try: + existing = self.includes[name] + except KeyError: + pass + else: + if existing.condition and not condition: + # If the previous include has a condition and the new one is + # unconditional, override the include. + pass + else: + # Already included, do nothing. Only mention a single reason, + # no need to list all of them. + return + + self.includes[name] = Include(name, reason, condition) + + def add_destination( + self, + name: str, + type: str, + *args: str + ) -> None: + if name in self.destinations: + fail(f"Destination already exists: {name!r}") + self.destinations[name] = Destination(name, type, self, args) + + def get_destination(self, name: str) -> Destination: + d = self.destinations.get(name) + if not d: + fail(f"Destination does not exist: {name!r}") + return d + + def get_destination_buffer( + self, + name: str, + item: int = 0 + ) -> list[str]: + d = self.get_destination(name) + return d.buffers[item] + + def parse(self, input: str) -> str: + printer = self.printer + self.block_parser = BlockParser(input, self.language, verify=self.verify) + for block in self.block_parser: + dsl_name = block.dsl_name + if dsl_name: + if dsl_name not in self.parsers: + assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." + self.parsers[dsl_name] = parsers[dsl_name](self) + parser = self.parsers[dsl_name] + parser.parse(block) + printer.print_block(block, + limited_capi=self.limited_capi, + header_includes=self.includes) + + # these are destinations not buffers + for name, destination in self.destinations.items(): + if destination.type == 'suppress': + continue + output = destination.dump() + + if output: + block = Block("", dsl_name="clinic", output=output) + + if destination.type == 'buffer': + block.input = "dump " + name + "\n" + warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") + printer.write("\n") + printer.print_block(block, + limited_capi=self.limited_capi, + header_includes=self.includes) + continue + + if destination.type == 'file': + try: + dirname = os.path.dirname(destination.filename) + try: + os.makedirs(dirname) + except FileExistsError: + if not os.path.isdir(dirname): + fail(f"Can't write to destination " + f"{destination.filename!r}; " + f"can't make directory {dirname!r}!") + if self.verify: + with open(destination.filename) as f: + parser_2 = BlockParser(f.read(), language=self.language) + blocks = list(parser_2) + if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): + fail(f"Modified destination file " + f"{destination.filename!r}; not overwriting!") + except FileNotFoundError: + pass + + block.input = 'preserve\n' + printer_2 = BlockPrinter(self.language) + printer_2.print_block(block, + core_includes=True, + limited_capi=self.limited_capi, + header_includes=self.includes) + libclinic.write_file(destination.filename, + printer_2.f.getvalue()) + continue + + return printer.f.getvalue() + + def _module_and_class( + self, fields: Sequence[str] + ) -> tuple[Module | Clinic, Class | None]: + """ + fields should be an iterable of field names. + returns a tuple of (module, class). + the module object could actually be self (a clinic object). + this function is only ever used to find the parent of where + a new class/module should go. + """ + parent: Clinic | Module | Class = self + module: Clinic | Module = self + cls: Class | None = None + + for idx, field in enumerate(fields): + if not isinstance(parent, Class): + if field in parent.modules: + parent = module = parent.modules[field] + continue + if field in parent.classes: + parent = cls = parent.classes[field] + else: + fullname = ".".join(fields[idx:]) + fail(f"Parent class or module {fullname!r} does not exist.") + + return module, cls + + def __repr__(self) -> str: + return "" diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index 3f4ca4aab56d67..ed08d12d8bfb29 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -19,7 +19,7 @@ from libclinic.converters import ( defining_class_converter, object_converter, self_converter) if TYPE_CHECKING: - from clinic import Clinic + from libclinic.app import Clinic def declare_parser( diff --git a/Tools/clinic/libclinic/cli.py b/Tools/clinic/libclinic/cli.py new file mode 100644 index 00000000000000..f36c6d04efd383 --- /dev/null +++ b/Tools/clinic/libclinic/cli.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import argparse +import inspect +import os +import re +import sys +from collections.abc import Callable +from typing import NoReturn + + +# Local imports. +import libclinic +import libclinic.cpp +from libclinic import ClinicError +from libclinic.language import Language, PythonLanguage +from libclinic.block_parser import BlockParser +from libclinic.converter import ( + ConverterType, converters, legacy_converters) +from libclinic.return_converters import ( + return_converters, ReturnConverterType) +from libclinic.clanguage import CLanguage +from libclinic.app import Clinic + + +# TODO: +# +# soon: +# +# * allow mixing any two of {positional-only, positional-or-keyword, +# keyword-only} +# * dict constructor uses positional-only and keyword-only +# * max and min use positional only with an optional group +# and keyword-only +# + + +# Match '#define Py_LIMITED_API'. +# Match '# define Py_LIMITED_API 0x030d0000' (without the version). +LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API') + + +# "extensions" maps the file extension ("c", "py") to Language classes. +LangDict = dict[str, Callable[[str], Language]] +extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } +extensions['py'] = PythonLanguage + + +def parse_file( + filename: str, + *, + limited_capi: bool, + output: str | None = None, + verify: bool = True, +) -> None: + if not output: + output = filename + + extension = os.path.splitext(filename)[1][1:] + if not extension: + raise ClinicError(f"Can't extract file type for file {filename!r}") + + try: + language = extensions[extension](filename) + except KeyError: + raise ClinicError(f"Can't identify file type for file {filename!r}") + + with open(filename, encoding="utf-8") as f: + raw = f.read() + + # exit quickly if there are no clinic markers in the file + find_start_re = BlockParser("", language).find_start_re + if not find_start_re.search(raw): + return + + if LIMITED_CAPI_REGEX.search(raw): + limited_capi = True + + assert isinstance(language, CLanguage) + clinic = Clinic(language, + verify=verify, + filename=filename, + limited_capi=limited_capi) + cooked = clinic.parse(raw) + + libclinic.write_file(output, cooked) + + +def create_cli() -> argparse.ArgumentParser: + cmdline = argparse.ArgumentParser( + prog="clinic.py", + description="""Preprocessor for CPython C files. + +The purpose of the Argument Clinic is automating all the boilerplate involved +with writing argument parsing code for builtins and providing introspection +signatures ("docstrings") for CPython builtins. + +For more information see https://devguide.python.org/development-tools/clinic/""") + cmdline.add_argument("-f", "--force", action='store_true', + help="force output regeneration") + cmdline.add_argument("-o", "--output", type=str, + help="redirect file output to OUTPUT") + cmdline.add_argument("-v", "--verbose", action='store_true', + help="enable verbose mode") + cmdline.add_argument("--converters", action='store_true', + help=("print a list of all supported converters " + "and return converters")) + cmdline.add_argument("--make", action='store_true', + help="walk --srcdir to run over all relevant files") + cmdline.add_argument("--srcdir", type=str, default=os.curdir, + help="the directory tree to walk in --make mode") + cmdline.add_argument("--exclude", type=str, action="append", + help=("a file to exclude in --make mode; " + "can be given multiple times")) + cmdline.add_argument("--limited", dest="limited_capi", action='store_true', + help="use the Limited C API") + cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", + help="the list of files to process") + return cmdline + + +def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: + if ns.converters: + if ns.filename: + parser.error( + "can't specify --converters and a filename at the same time" + ) + AnyConverterType = ConverterType | ReturnConverterType + converter_list: list[tuple[str, AnyConverterType]] = [] + return_converter_list: list[tuple[str, AnyConverterType]] = [] + + for name, converter in converters.items(): + converter_list.append(( + name, + converter, + )) + for name, return_converter in return_converters.items(): + return_converter_list.append(( + name, + return_converter + )) + + print() + + print("Legacy converters:") + legacy = sorted(legacy_converters) + print(' ' + ' '.join(c for c in legacy if c[0].isupper())) + print(' ' + ' '.join(c for c in legacy if c[0].islower())) + print() + + for title, attribute, ids in ( + ("Converters", 'converter_init', converter_list), + ("Return converters", 'return_converter_init', return_converter_list), + ): + print(title + ":") + + ids.sort(key=lambda item: item[0].lower()) + longest = -1 + for name, _ in ids: + longest = max(longest, len(name)) + + for name, cls in ids: + callable = getattr(cls, attribute, None) + if not callable: + continue + signature = inspect.signature(callable) + parameters = [] + for parameter_name, parameter in signature.parameters.items(): + if parameter.kind == inspect.Parameter.KEYWORD_ONLY: + if parameter.default != inspect.Parameter.empty: + s = f'{parameter_name}={parameter.default!r}' + else: + s = parameter_name + parameters.append(s) + print(' {}({})'.format(name, ', '.join(parameters))) + print() + print("All converters also accept (c_default=None, py_default=None, annotation=None).") + print("All return converters also accept (py_default=None).") + return + + if ns.make: + if ns.output or ns.filename: + parser.error("can't use -o or filenames with --make") + if not ns.srcdir: + parser.error("--srcdir must not be empty with --make") + if ns.exclude: + excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude] + excludes = [os.path.normpath(f) for f in excludes] + else: + excludes = [] + for root, dirs, files in os.walk(ns.srcdir): + for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): + if rcs_dir in dirs: + dirs.remove(rcs_dir) + for filename in files: + # handle .c, .cpp and .h files + if not filename.endswith(('.c', '.cpp', '.h')): + continue + path = os.path.join(root, filename) + path = os.path.normpath(path) + if path in excludes: + continue + if ns.verbose: + print(path) + parse_file(path, + verify=not ns.force, limited_capi=ns.limited_capi) + return + + if not ns.filename: + parser.error("no input files") + + if ns.output and len(ns.filename) > 1: + parser.error("can't use -o with multiple filenames") + + for filename in ns.filename: + if ns.verbose: + print(filename) + parse_file(filename, output=ns.output, + verify=not ns.force, limited_capi=ns.limited_capi) + + +def main(argv: list[str] | None = None) -> NoReturn: + parser = create_cli() + args = parser.parse_args(argv) + try: + run_clinic(parser, args) + except ClinicError as exc: + sys.stderr.write(exc.report()) + sys.exit(1) + else: + sys.exit(0) diff --git a/Tools/clinic/libclinic/codegen.py b/Tools/clinic/libclinic/codegen.py index 097fff0edf38c6..ad08e22e2e1c2c 100644 --- a/Tools/clinic/libclinic/codegen.py +++ b/Tools/clinic/libclinic/codegen.py @@ -3,14 +3,14 @@ import io import os from typing import Final, TYPE_CHECKING -if TYPE_CHECKING: - from clinic import Clinic import libclinic from libclinic import fail from libclinic.crenderdata import Include from libclinic.language import Language from libclinic.block_parser import Block +if TYPE_CHECKING: + from libclinic.app import Clinic @dc.dataclass(slots=True) @@ -185,3 +185,6 @@ def clear(self) -> None: def dump(self) -> str: return self.buffers.dump() + + +DestinationDict = dict[str, Destination] diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py new file mode 100644 index 00000000000000..4c739efe1066e4 --- /dev/null +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -0,0 +1,1592 @@ +from __future__ import annotations +import ast +import enum +import inspect +import pprint +import re +import shlex +import sys +from collections.abc import Callable +from types import FunctionType, NoneType +from typing import TYPE_CHECKING, Any, NamedTuple + +import libclinic +from libclinic import ( + ClinicError, VersionTuple, + fail, warn, unspecified, unknown, NULL) +from libclinic.function import ( + Module, Class, Function, Parameter, + FunctionKind, + CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, + GETTER, SETTER) +from libclinic.converter import ( + converters, legacy_converters) +from libclinic.converters import ( + self_converter, defining_class_converter, + correct_name_for_self) +from libclinic.return_converters import ( + CReturnConverter, return_converters, + int_return_converter) +from libclinic.parser import create_parser_namespace +if TYPE_CHECKING: + from libclinic.block_parser import Block + from libclinic.app import Clinic + + +unsupported_special_methods: set[str] = set(""" + +__abs__ +__add__ +__and__ +__call__ +__delitem__ +__divmod__ +__eq__ +__float__ +__floordiv__ +__ge__ +__getattr__ +__getattribute__ +__getitem__ +__gt__ +__hash__ +__iadd__ +__iand__ +__ifloordiv__ +__ilshift__ +__imatmul__ +__imod__ +__imul__ +__index__ +__int__ +__invert__ +__ior__ +__ipow__ +__irshift__ +__isub__ +__iter__ +__itruediv__ +__ixor__ +__le__ +__len__ +__lshift__ +__lt__ +__matmul__ +__mod__ +__mul__ +__neg__ +__next__ +__or__ +__pos__ +__pow__ +__radd__ +__rand__ +__rdivmod__ +__repr__ +__rfloordiv__ +__rlshift__ +__rmatmul__ +__rmod__ +__rmul__ +__ror__ +__rpow__ +__rrshift__ +__rshift__ +__rsub__ +__rtruediv__ +__rxor__ +__setattr__ +__setitem__ +__str__ +__sub__ +__truediv__ +__xor__ + +""".strip().split()) + + +StateKeeper = Callable[[str], None] +ConverterArgs = dict[str, Any] + + +class ParamState(enum.IntEnum): + """Parameter parsing state. + + [ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] <- line + 01 2 3 4 5 6 <- state transitions + """ + # Before we've seen anything. + # Legal transitions: to LEFT_SQUARE_BEFORE or REQUIRED + START = 0 + + # Left square backets before required params. + LEFT_SQUARE_BEFORE = 1 + + # In a group, before required params. + GROUP_BEFORE = 2 + + # Required params, positional-or-keyword or positional-only (we + # don't know yet). Renumber left groups! + REQUIRED = 3 + + # Positional-or-keyword or positional-only params that now must have + # default values. + OPTIONAL = 4 + + # In a group, after required params. + GROUP_AFTER = 5 + + # Right square brackets after required params. + RIGHT_SQUARE_AFTER = 6 + + +class FunctionNames(NamedTuple): + full_name: str + c_basename: str + + +def eval_ast_expr( + node: ast.expr, + *, + filename: str = '-' +) -> Any: + """ + Takes an ast.Expr node. Compiles it into a function object, + then calls the function object with 0 arguments. + Returns the result of that function call. + + globals represents the globals dict the expression + should see. (There's no equivalent for "locals" here.) + """ + + if isinstance(node, ast.Expr): + node = node.value + + expr = ast.Expression(node) + namespace = create_parser_namespace() + co = compile(expr, filename, 'eval') + fn = FunctionType(co, namespace) + return fn() + + +class IndentStack: + def __init__(self) -> None: + self.indents: list[int] = [] + self.margin: str | None = None + + def _ensure(self) -> None: + if not self.indents: + fail('IndentStack expected indents, but none are defined.') + + def measure(self, line: str) -> int: + """ + Returns the length of the line's margin. + """ + if '\t' in line: + fail('Tab characters are illegal in the Argument Clinic DSL.') + stripped = line.lstrip() + if not len(stripped): + # we can't tell anything from an empty line + # so just pretend it's indented like our current indent + self._ensure() + return self.indents[-1] + return len(line) - len(stripped) + + def infer(self, line: str) -> int: + """ + Infer what is now the current margin based on this line. + Returns: + 1 if we have indented (or this is the first margin) + 0 if the margin has not changed + -N if we have dedented N times + """ + indent = self.measure(line) + margin = ' ' * indent + if not self.indents: + self.indents.append(indent) + self.margin = margin + return 1 + current = self.indents[-1] + if indent == current: + return 0 + if indent > current: + self.indents.append(indent) + self.margin = margin + return 1 + # indent < current + if indent not in self.indents: + fail("Illegal outdent.") + outdent_count = 0 + while indent != current: + self.indents.pop() + current = self.indents[-1] + outdent_count -= 1 + self.margin = margin + return outdent_count + + @property + def depth(self) -> int: + """ + Returns how many margins are currently defined. + """ + return len(self.indents) + + def dedent(self, line: str) -> str: + """ + Dedents a line by the currently defined margin. + """ + assert self.margin is not None, "Cannot call .dedent() before calling .infer()" + margin = self.margin + indent = self.indents[-1] + if not line.startswith(margin): + fail('Cannot dedent; line does not start with the previous margin.') + return line[indent:] + + +class DSLParser: + function: Function | None + state: StateKeeper + keyword_only: bool + positional_only: bool + deprecated_positional: VersionTuple | None + deprecated_keyword: VersionTuple | None + group: int + parameter_state: ParamState + indent: IndentStack + kind: FunctionKind + coexist: bool + forced_text_signature: str | None + parameter_continuation: str + preserve_output: bool + critical_section: bool + target_critical_section: list[str] + from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') + + def __init__(self, clinic: Clinic) -> None: + self.clinic = clinic + + self.directives = {} + for name in dir(self): + # functions that start with directive_ are added to directives + _, s, key = name.partition("directive_") + if s: + self.directives[key] = getattr(self, name) + + # functions that start with at_ are too, with an @ in front + _, s, key = name.partition("at_") + if s: + self.directives['@' + key] = getattr(self, name) + + self.reset() + + def reset(self) -> None: + self.function = None + self.state = self.state_dsl_start + self.keyword_only = False + self.positional_only = False + self.deprecated_positional = None + self.deprecated_keyword = None + self.group = 0 + self.parameter_state: ParamState = ParamState.START + self.indent = IndentStack() + self.kind = CALLABLE + self.coexist = False + self.forced_text_signature = None + self.parameter_continuation = '' + self.preserve_output = False + self.critical_section = False + self.target_critical_section = [] + + def directive_module(self, name: str) -> None: + fields = name.split('.')[:-1] + module, cls = self.clinic._module_and_class(fields) + if cls: + fail("Can't nest a module inside a class!") + + if name in module.modules: + fail(f"Already defined module {name!r}!") + + m = Module(name, module) + module.modules[name] = m + self.block.signatures.append(m) + + def directive_class( + self, + name: str, + typedef: str, + type_object: str + ) -> None: + fields = name.split('.') + name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + parent = cls or module + if name in parent.classes: + fail(f"Already defined class {name!r}!") + + c = Class(name, module, cls, typedef, type_object) + parent.classes[name] = c + self.block.signatures.append(c) + + def directive_set(self, name: str, value: str) -> None: + if name not in ("line_prefix", "line_suffix"): + fail(f"unknown variable {name!r}") + + value = value.format_map({ + 'block comment start': '/*', + 'block comment end': '*/', + }) + + self.clinic.__dict__[name] = value + + def directive_destination( + self, + name: str, + command: str, + *args: str + ) -> None: + match command: + case "new": + self.clinic.add_destination(name, *args) + case "clear": + self.clinic.get_destination(name).clear() + case _: + fail(f"unknown destination command {command!r}") + + + def directive_output( + self, + command_or_name: str, + destination: str = '' + ) -> None: + fd = self.clinic.destination_buffers + + if command_or_name == "preset": + preset = self.clinic.presets.get(destination) + if not preset: + fail(f"Unknown preset {destination!r}!") + fd.update(preset) + return + + if command_or_name == "push": + self.clinic.destination_buffers_stack.append(fd.copy()) + return + + if command_or_name == "pop": + if not self.clinic.destination_buffers_stack: + fail("Can't 'output pop', stack is empty!") + previous_fd = self.clinic.destination_buffers_stack.pop() + fd.update(previous_fd) + return + + # secret command for debugging! + if command_or_name == "print": + self.block.output.append(pprint.pformat(fd)) + self.block.output.append('\n') + return + + d = self.clinic.get_destination_buffer(destination) + + if command_or_name == "everything": + for name in list(fd): + fd[name] = d + return + + if command_or_name not in fd: + allowed = ["preset", "push", "pop", "print", "everything"] + allowed.extend(fd) + fail(f"Invalid command or destination name {command_or_name!r}. " + "Must be one of:\n -", + "\n - ".join([repr(word) for word in allowed])) + fd[command_or_name] = d + + def directive_dump(self, name: str) -> None: + self.block.output.append(self.clinic.get_destination(name).dump()) + + def directive_printout(self, *args: str) -> None: + self.block.output.append(' '.join(args)) + self.block.output.append('\n') + + def directive_preserve(self) -> None: + if self.preserve_output: + fail("Can't have 'preserve' twice in one block!") + self.preserve_output = True + + def at_classmethod(self) -> None: + if self.kind is not CALLABLE: + fail("Can't set @classmethod, function is not a normal callable") + self.kind = CLASS_METHOD + + def at_critical_section(self, *args: str) -> None: + if len(args) > 2: + fail("Up to 2 critical section variables are supported") + self.target_critical_section.extend(args) + self.critical_section = True + + def at_getter(self) -> None: + match self.kind: + case FunctionKind.GETTER: + fail("Cannot apply @getter twice to the same function!") + case FunctionKind.SETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.GETTER + + def at_setter(self) -> None: + match self.kind: + case FunctionKind.SETTER: + fail("Cannot apply @setter twice to the same function!") + case FunctionKind.GETTER: + fail("Cannot apply both @getter and @setter to the same function!") + case _: + self.kind = FunctionKind.SETTER + + def at_staticmethod(self) -> None: + if self.kind is not CALLABLE: + fail("Can't set @staticmethod, function is not a normal callable") + self.kind = STATIC_METHOD + + def at_coexist(self) -> None: + if self.coexist: + fail("Called @coexist twice!") + self.coexist = True + + def at_text_signature(self, text_signature: str) -> None: + if self.forced_text_signature: + fail("Called @text_signature twice!") + self.forced_text_signature = text_signature + + def parse(self, block: Block) -> None: + self.reset() + self.block = block + self.saved_output = self.block.output + block.output = [] + block_start = self.clinic.block_parser.line_number + lines = block.input.split('\n') + for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): + if '\t' in line: + fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', + line_number=block_start) + try: + self.state(line) + except ClinicError as exc: + exc.lineno = line_number + exc.filename = self.clinic.filename + raise + + self.do_post_block_processing_cleanup(line_number) + block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) + + if self.preserve_output: + if block.output: + fail("'preserve' only works for blocks that don't produce any output!", + line_number=line_number) + block.output = self.saved_output + + def in_docstring(self) -> bool: + """Return true if we are processing a docstring.""" + return self.state in { + self.state_parameter_docstring, + self.state_function_docstring, + } + + def valid_line(self, line: str) -> bool: + # ignore comment-only lines + if line.lstrip().startswith('#'): + return False + + # Ignore empty lines too + # (but not in docstring sections!) + if not self.in_docstring() and not line.strip(): + return False + + return True + + def next( + self, + state: StateKeeper, + line: str | None = None + ) -> None: + self.state = state + if line is not None: + self.state(line) + + def state_dsl_start(self, line: str) -> None: + if not self.valid_line(line): + return + + # is it a directive? + fields = shlex.split(line) + directive_name = fields[0] + directive = self.directives.get(directive_name, None) + if directive: + try: + directive(*fields[1:]) + except TypeError as e: + fail(str(e)) + return + + self.next(self.state_modulename_name, line) + + def parse_function_names(self, line: str) -> FunctionNames: + left, as_, right = line.partition(' as ') + full_name = left.strip() + c_basename = right.strip() + if as_ and not c_basename: + fail("No C basename provided after 'as' keyword") + if not c_basename: + fields = full_name.split(".") + if fields[-1] == '__new__': + fields.pop() + c_basename = "_".join(fields) + if not libclinic.is_legal_py_identifier(full_name): + fail(f"Illegal function name: {full_name!r}") + if not libclinic.is_legal_c_identifier(c_basename): + fail(f"Illegal C basename: {c_basename!r}") + names = FunctionNames(full_name=full_name, c_basename=c_basename) + self.normalize_function_kind(names.full_name) + return names + + def normalize_function_kind(self, fullname: str) -> None: + # Fetch the method name and possibly class. + fields = fullname.split('.') + name = fields.pop() + _, cls = self.clinic._module_and_class(fields) + + # Check special method requirements. + if name in unsupported_special_methods: + fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") + if name == '__init__' and (self.kind is not CALLABLE or not cls): + fail(f"{name!r} must be a normal method; got '{self.kind}'!") + if name == '__new__' and (self.kind is not CLASS_METHOD or not cls): + fail("'__new__' must be a class method!") + if self.kind in {GETTER, SETTER} and not cls: + fail("@getter and @setter must be methods") + + # Normalise self.kind. + if name == '__new__': + self.kind = METHOD_NEW + elif name == '__init__': + self.kind = METHOD_INIT + + def resolve_return_converter( + self, full_name: str, forced_converter: str + ) -> CReturnConverter: + if forced_converter: + if self.kind in {GETTER, SETTER}: + fail(f"@{self.kind.name.lower()} method cannot define a return type") + if self.kind is METHOD_INIT: + fail("__init__ methods cannot define a return type") + ast_input = f"def x() -> {forced_converter}: pass" + try: + module_node = ast.parse(ast_input) + except SyntaxError: + fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") + function_node = module_node.body[0] + assert isinstance(function_node, ast.FunctionDef) + try: + name, legacy, kwargs = self.parse_converter(function_node.returns) + if legacy: + fail(f"Legacy converter {name!r} not allowed as a return converter") + if name not in return_converters: + fail(f"No available return converter called {name!r}") + return return_converters[name](**kwargs) + except ValueError: + fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") + + if self.kind in {METHOD_INIT, SETTER}: + return int_return_converter() + return CReturnConverter() + + def parse_cloned_function(self, names: FunctionNames, existing: str) -> None: + full_name, c_basename = names + fields = [x.strip() for x in existing.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + parent = cls or module + + for existing_function in parent.functions: + if existing_function.name == function_name: + break + else: + print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) + print(f"{(cls or module).functions=}", file=sys.stderr) + fail(f"Couldn't find existing function {existing!r}!") + + fields = [x.strip() for x in full_name.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + overrides: dict[str, Any] = { + "name": function_name, + "full_name": full_name, + "module": module, + "cls": cls, + "c_basename": c_basename, + "docstring": "", + } + if not (existing_function.kind is self.kind and + existing_function.coexist == self.coexist): + # Allow __new__ or __init__ methods. + if existing_function.kind.new_or_init: + overrides["kind"] = self.kind + # Future enhancement: allow custom return converters + overrides["return_converter"] = CReturnConverter() + else: + fail("'kind' of function and cloned function don't match! " + "(@classmethod/@staticmethod/@coexist)") + function = existing_function.copy(**overrides) + self.function = function + self.block.signatures.append(function) + (cls or module).functions.append(function) + self.next(self.state_function_docstring) + + def state_modulename_name(self, line: str) -> None: + # looking for declaration, which establishes the leftmost column + # line should be + # modulename.fnname [as c_basename] [-> return annotation] + # square brackets denote optional syntax. + # + # alternatively: + # modulename.fnname [as c_basename] = modulename.existing_fn_name + # clones the parameters and return converter from that + # function. you can't modify them. you must enter a + # new docstring. + # + # (but we might find a directive first!) + # + # this line is permitted to start with whitespace. + # we'll call this number of spaces F (for "function"). + + assert self.valid_line(line) + self.indent.infer(line) + + # are we cloning? + before, equals, existing = line.rpartition('=') + if equals: + existing = existing.strip() + if libclinic.is_legal_py_identifier(existing): + # we're cloning! + names = self.parse_function_names(before) + return self.parse_cloned_function(names, existing) + + line, _, returns = line.partition('->') + returns = returns.strip() + full_name, c_basename = self.parse_function_names(line) + return_converter = self.resolve_return_converter(full_name, returns) + + fields = [x.strip() for x in full_name.split('.')] + function_name = fields.pop() + module, cls = self.clinic._module_and_class(fields) + + func = Function( + name=function_name, + full_name=full_name, + module=module, + cls=cls, + c_basename=c_basename, + return_converter=return_converter, + kind=self.kind, + coexist=self.coexist, + critical_section=self.critical_section, + target_critical_section=self.target_critical_section + ) + self.add_function(func) + + self.next(self.state_parameters_start) + + def add_function(self, func: Function) -> None: + # Insert a self converter automatically. + tp, name = correct_name_for_self(func) + if func.cls and tp == "PyObject *": + func.self_converter = self_converter(name, name, func, + type=func.cls.typedef) + else: + func.self_converter = self_converter(name, name, func) + func.parameters[name] = Parameter( + name, + inspect.Parameter.POSITIONAL_ONLY, + function=func, + converter=func.self_converter + ) + + self.block.signatures.append(func) + self.function = func + (func.cls or func.module).functions.append(func) + + # Now entering the parameters section. The rules, formally stated: + # + # * All lines must be indented with spaces only. + # * The first line must be a parameter declaration. + # * The first line must be indented. + # * This first line establishes the indent for parameters. + # * We'll call this number of spaces P (for "parameter"). + # * Thenceforth: + # * Lines indented with P spaces specify a parameter. + # * Lines indented with > P spaces are docstrings for the previous + # parameter. + # * We'll call this number of spaces D (for "docstring"). + # * All subsequent lines indented with >= D spaces are stored as + # part of the per-parameter docstring. + # * All lines will have the first D spaces of the indent stripped + # before they are stored. + # * It's illegal to have a line starting with a number of spaces X + # such that P < X < D. + # * A line with < P spaces is the first line of the function + # docstring, which ends processing for parameters and per-parameter + # docstrings. + # * The first line of the function docstring must be at the same + # indent as the function declaration. + # * It's illegal to have any line in the parameters section starting + # with X spaces such that F < X < P. (As before, F is the indent + # of the function declaration.) + # + # Also, currently Argument Clinic places the following restrictions on groups: + # * Each group must contain at least one parameter. + # * Each group may contain at most one group, which must be the furthest + # thing in the group from the required parameters. (The nested group + # must be the first in the group when it's before the required + # parameters, and the last thing in the group when after the required + # parameters.) + # * There may be at most one (top-level) group to the left or right of + # the required parameters. + # * You must specify a slash, and it must be after all parameters. + # (In other words: either all parameters are positional-only, + # or none are.) + # + # Said another way: + # * Each group must contain at least one parameter. + # * All left square brackets before the required parameters must be + # consecutive. (You can't have a left square bracket followed + # by a parameter, then another left square bracket. You can't + # have a left square bracket, a parameter, a right square bracket, + # and then a left square bracket.) + # * All right square brackets after the required parameters must be + # consecutive. + # + # These rules are enforced with a single state variable: + # "parameter_state". (Previously the code was a miasma of ifs and + # separate boolean state variables.) The states are defined in the + # ParamState class. + + def state_parameters_start(self, line: str) -> None: + if not self.valid_line(line): + return + + # if this line is not indented, we have no parameters + if not self.indent.infer(line): + return self.next(self.state_function_docstring, line) + + assert self.function is not None + if self.function.kind in {GETTER, SETTER}: + getset = self.function.kind.name.lower() + fail(f"@{getset} methods cannot define parameters") + + self.parameter_continuation = '' + return self.next(self.state_parameter, line) + + + def to_required(self) -> None: + """ + Transition to the "required" parameter state. + """ + if self.parameter_state is not ParamState.REQUIRED: + self.parameter_state = ParamState.REQUIRED + assert self.function is not None + for p in self.function.parameters.values(): + p.group = -p.group + + def state_parameter(self, line: str) -> None: + assert isinstance(self.function, Function) + + if not self.valid_line(line): + return + + if self.parameter_continuation: + line = self.parameter_continuation + ' ' + line.lstrip() + self.parameter_continuation = '' + + assert self.indent.depth == 2 + indent = self.indent.infer(line) + if indent == -1: + # we outdented, must be to definition column + return self.next(self.state_function_docstring, line) + + if indent == 1: + # we indented, must be to new parameter docstring column + return self.next(self.state_parameter_docstring_start, line) + + line = line.rstrip() + if line.endswith('\\'): + self.parameter_continuation = line[:-1] + return + + line = line.lstrip() + version: VersionTuple | None = None + match = self.from_version_re.fullmatch(line) + if match: + line = match[1] + version = self.parse_version(match[2]) + + func = self.function + match line: + case '*': + self.parse_star(func, version) + case '[': + self.parse_opening_square_bracket(func) + case ']': + self.parse_closing_square_bracket(func) + case '/': + self.parse_slash(func, version) + case param: + self.parse_parameter(param) + + def parse_parameter(self, line: str) -> None: + assert self.function is not None + + match self.parameter_state: + case ParamState.START | ParamState.REQUIRED: + self.to_required() + case ParamState.LEFT_SQUARE_BEFORE: + self.parameter_state = ParamState.GROUP_BEFORE + case ParamState.GROUP_BEFORE: + if not self.group: + self.to_required() + case ParamState.GROUP_AFTER | ParamState.OPTIONAL: + pass + case st: + fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.a)") + + # handle "as" for parameters too + c_name = None + name, have_as_token, trailing = line.partition(' as ') + if have_as_token: + name = name.strip() + if ' ' not in name: + fields = trailing.strip().split(' ') + if not fields: + fail("Invalid 'as' clause!") + c_name = fields[0] + if c_name.endswith(':'): + name += ':' + c_name = c_name[:-1] + fields[0] = name + line = ' '.join(fields) + + default: str | None + base, equals, default = line.rpartition('=') + if not equals: + base = default + default = None + + module = None + try: + ast_input = f"def x({base}): pass" + module = ast.parse(ast_input) + except SyntaxError: + try: + # the last = was probably inside a function call, like + # c: int(accept={str}) + # so assume there was no actual default value. + default = None + ast_input = f"def x({line}): pass" + module = ast.parse(ast_input) + except SyntaxError: + pass + if not module: + fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", + repr(line)) + + function = module.body[0] + assert isinstance(function, ast.FunctionDef) + function_args = function.args + + if len(function_args.args) > 1: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (comma?): {line!r}") + if function_args.defaults or function_args.kw_defaults: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (default value?): {line!r}") + if function_args.kwarg: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (**kwargs?): {line!r}") + + if function_args.vararg: + if any(p.is_vararg() for p in self.function.parameters.values()): + fail("Too many var args") + is_vararg = True + parameter = function_args.vararg + else: + is_vararg = False + parameter = function_args.args[0] + + parameter_name = parameter.arg + name, legacy, kwargs = self.parse_converter(parameter.annotation) + + value: object + if not default: + if self.parameter_state is ParamState.OPTIONAL: + fail(f"Can't have a parameter without a default ({parameter_name!r}) " + "after a parameter with a default!") + if is_vararg: + value = NULL + kwargs.setdefault('c_default', "NULL") + else: + value = unspecified + if 'py_default' in kwargs: + fail("You can't specify py_default without specifying a default value!") + else: + if is_vararg: + fail("Vararg can't take a default value!") + + if self.parameter_state is ParamState.REQUIRED: + self.parameter_state = ParamState.OPTIONAL + default = default.strip() + bad = False + ast_input = f"x = {default}" + try: + module = ast.parse(ast_input) + + if 'c_default' not in kwargs: + # we can only represent very simple data values in C. + # detect whether default is okay, via a denylist + # of disallowed ast nodes. + class DetectBadNodes(ast.NodeVisitor): + bad = False + def bad_node(self, node: ast.AST) -> None: + self.bad = True + + # inline function call + visit_Call = bad_node + # inline if statement ("x = 3 if y else z") + visit_IfExp = bad_node + + # comprehensions and generator expressions + visit_ListComp = visit_SetComp = bad_node + visit_DictComp = visit_GeneratorExp = bad_node + + # literals for advanced types + visit_Dict = visit_Set = bad_node + visit_List = visit_Tuple = bad_node + + # "starred": "a = [1, 2, 3]; *a" + visit_Starred = bad_node + + denylist = DetectBadNodes() + denylist.visit(module) + bad = denylist.bad + else: + # if they specify a c_default, we can be more lenient about the default value. + # but at least make an attempt at ensuring it's a valid expression. + try: + value = eval(default) + except NameError: + pass # probably a named constant + except Exception as e: + fail("Malformed expression given as default value " + f"{default!r} caused {e!r}") + else: + if value is unspecified: + fail("'unspecified' is not a legal default value!") + if bad: + fail(f"Unsupported expression as default value: {default!r}") + + assignment = module.body[0] + assert isinstance(assignment, ast.Assign) + expr = assignment.value + # mild hack: explicitly support NULL as a default value + c_default: str | None + if isinstance(expr, ast.Name) and expr.id == 'NULL': + value = NULL + py_default = '' + c_default = "NULL" + elif (isinstance(expr, ast.BinOp) or + (isinstance(expr, ast.UnaryOp) and + not (isinstance(expr.operand, ast.Constant) and + type(expr.operand.value) in {int, float, complex}) + )): + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail(f"When you specify an expression ({default!r}) " + f"as your default value, " + f"you MUST specify a valid c_default.", + ast.dump(expr)) + py_default = default + value = unknown + elif isinstance(expr, ast.Attribute): + a = [] + n: ast.expr | ast.Attribute = expr + while isinstance(n, ast.Attribute): + a.append(n.attr) + n = n.value + if not isinstance(n, ast.Name): + fail(f"Unsupported default value {default!r} " + "(looked like a Python constant)") + a.append(n.id) + py_default = ".".join(reversed(a)) + + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail(f"When you specify a named constant ({py_default!r}) " + "as your default value, " + "you MUST specify a valid c_default.") + + try: + value = eval(py_default) + except NameError: + value = unknown + else: + value = ast.literal_eval(expr) + py_default = repr(value) + if isinstance(value, (bool, NoneType)): + c_default = "Py_" + py_default + elif isinstance(value, str): + c_default = libclinic.c_repr(value) + else: + c_default = py_default + + except SyntaxError as e: + fail(f"Syntax error: {e.text!r}") + except (ValueError, AttributeError): + value = unknown + c_default = kwargs.get("c_default") + py_default = default + if not (isinstance(c_default, str) and c_default): + fail("When you specify a named constant " + f"({py_default!r}) as your default value, " + "you MUST specify a valid c_default.") + + kwargs.setdefault('c_default', c_default) + kwargs.setdefault('py_default', py_default) + + dict = legacy_converters if legacy else converters + legacy_str = "legacy " if legacy else "" + if name not in dict: + fail(f'{name!r} is not a valid {legacy_str}converter') + # if you use a c_name for the parameter, we just give that name to the converter + # but the parameter object gets the python name + converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) + + kind: inspect._ParameterKind + if is_vararg: + kind = inspect.Parameter.VAR_POSITIONAL + elif self.keyword_only: + kind = inspect.Parameter.KEYWORD_ONLY + else: + kind = inspect.Parameter.POSITIONAL_OR_KEYWORD + + if isinstance(converter, self_converter): + if len(self.function.parameters) == 1: + if self.parameter_state is not ParamState.REQUIRED: + fail("A 'self' parameter cannot be marked optional.") + if value is not unspecified: + fail("A 'self' parameter cannot have a default value.") + if self.group: + fail("A 'self' parameter cannot be in an optional group.") + kind = inspect.Parameter.POSITIONAL_ONLY + self.parameter_state = ParamState.START + self.function.parameters.clear() + else: + fail("A 'self' parameter, if specified, must be the " + "very first thing in the parameter block.") + + if isinstance(converter, defining_class_converter): + _lp = len(self.function.parameters) + if _lp == 1: + if self.parameter_state is not ParamState.REQUIRED: + fail("A 'defining_class' parameter cannot be marked optional.") + if value is not unspecified: + fail("A 'defining_class' parameter cannot have a default value.") + if self.group: + fail("A 'defining_class' parameter cannot be in an optional group.") + else: + fail("A 'defining_class' parameter, if specified, must either " + "be the first thing in the parameter block, or come just " + "after 'self'.") + + + p = Parameter(parameter_name, kind, function=self.function, + converter=converter, default=value, group=self.group, + deprecated_positional=self.deprecated_positional) + + names = [k.name for k in self.function.parameters.values()] + if parameter_name in names[1:]: + fail(f"You can't have two parameters named {parameter_name!r}!") + elif names and parameter_name == names[0] and c_name is None: + fail(f"Parameter {parameter_name!r} requires a custom C name") + + key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name + self.function.parameters[key] = p + + @staticmethod + def parse_converter( + annotation: ast.expr | None + ) -> tuple[str, bool, ConverterArgs]: + match annotation: + case ast.Constant(value=str() as value): + return value, True, {} + case ast.Name(name): + return name, False, {} + case ast.Call(func=ast.Name(name)): + kwargs: ConverterArgs = {} + for node in annotation.keywords: + if not isinstance(node.arg, str): + fail("Cannot use a kwarg splat in a function-call annotation") + kwargs[node.arg] = eval_ast_expr(node.value) + return name, False, kwargs + case _: + fail( + "Annotations must be either a name, a function call, or a string." + ) + + def parse_version(self, thenceforth: str) -> VersionTuple: + """Parse Python version in `[from ...]` marker.""" + assert isinstance(self.function, Function) + + try: + major, minor = thenceforth.split(".") + return int(major), int(minor) + except ValueError: + fail( + f"Function {self.function.name!r}: expected format '[from major.minor]' " + f"where 'major' and 'minor' are integers; got {thenceforth!r}" + ) + + def parse_star(self, function: Function, version: VersionTuple | None) -> None: + """Parse keyword-only parameter marker '*'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + if self.keyword_only: + fail(f"Function {function.name!r} uses '*' more than once.") + self.check_previous_star() + self.check_remaining_star() + self.keyword_only = True + else: + if self.keyword_only: + fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") + if self.deprecated_positional: + if self.deprecated_positional == version: + fail(f"Function {function.name!r} uses '* [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_positional < version: + fail(f"Function {function.name!r}: '* [from " + f"{version[0]}.{version[1]}]' must precede '* [from " + f"{self.deprecated_positional[0]}.{self.deprecated_positional[1]}]'") + self.deprecated_positional = version + + def parse_opening_square_bracket(self, function: Function) -> None: + """Parse opening parameter group symbol '['.""" + match self.parameter_state: + case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: + self.parameter_state = ParamState.LEFT_SQUARE_BEFORE + case ParamState.REQUIRED | ParamState.GROUP_AFTER: + self.parameter_state = ParamState.GROUP_AFTER + case st: + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " + f"(Unexpected state {st}.b)") + self.group += 1 + function.docstring_only = True + + def parse_closing_square_bracket(self, function: Function) -> None: + """Parse closing parameter group symbol ']'.""" + if not self.group: + fail(f"Function {function.name!r} has a ']' without a matching '['.") + if not any(p.group == self.group for p in function.parameters.values()): + fail(f"Function {function.name!r} has an empty group. " + "All groups must contain at least one parameter.") + self.group -= 1 + match self.parameter_state: + case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: + self.parameter_state = ParamState.GROUP_BEFORE + case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: + self.parameter_state = ParamState.RIGHT_SQUARE_AFTER + case st: + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " + f"(Unexpected state {st}.c)") + + def parse_slash(self, function: Function, version: VersionTuple | None) -> None: + """Parse positional-only parameter marker '/'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + if self.deprecated_keyword: + fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/' must precede '*'") + if self.positional_only: + fail(f"Function {function.name!r} uses '/' more than once.") + else: + if self.deprecated_keyword: + if self.deprecated_keyword == version: + fail(f"Function {function.name!r} uses '/ [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_keyword > version: + fail(f"Function {function.name!r}: '/ [from " + f"{version[0]}.{version[1]}]' must precede '/ [from " + f"{self.deprecated_keyword[0]}.{self.deprecated_keyword[1]}]'") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '*'") + self.positional_only = True + self.deprecated_keyword = version + if version is not None: + found = False + for p in reversed(function.parameters.values()): + found = p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + break + if not found: + fail(f"Function {function.name!r} specifies '/ [from ...]' " + f"without preceding parameters.") + # REQUIRED and OPTIONAL are allowed here, that allows positional-only + # without option groups to work (and have default values!) + allowed = { + ParamState.REQUIRED, + ParamState.OPTIONAL, + ParamState.RIGHT_SQUARE_AFTER, + ParamState.GROUP_BEFORE, + } + if (self.parameter_state not in allowed) or self.group: + fail(f"Function {function.name!r} has an unsupported group configuration. " + f"(Unexpected state {self.parameter_state}.d)") + # fixup preceding parameters + for p in function.parameters.values(): + if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: + if version is None: + p.kind = inspect.Parameter.POSITIONAL_ONLY + elif p.deprecated_keyword is None: + p.deprecated_keyword = version + + def state_parameter_docstring_start(self, line: str) -> None: + assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" + self.parameter_docstring_indent = len(self.indent.margin) + assert self.indent.depth == 3 + return self.next(self.state_parameter_docstring, line) + + def docstring_append(self, obj: Function | Parameter, line: str) -> None: + """Add a rstripped line to the current docstring.""" + # gh-80282: We filter out non-ASCII characters from the docstring, + # since historically, some compilers may balk on non-ASCII input. + # If you're using Argument Clinic in an external project, + # you may not need to support the same array of platforms as CPython, + # so you may be able to remove this restriction. + matches = re.finditer(r'[^\x00-\x7F]', line) + if offending := ", ".join([repr(m[0]) for m in matches]): + warn("Non-ascii characters are not allowed in docstrings:", + offending) + + docstring = obj.docstring + if docstring: + docstring += "\n" + if stripped := line.rstrip(): + docstring += self.indent.dedent(stripped) + obj.docstring = docstring + + # every line of the docstring must start with at least F spaces, + # where F > P. + # these F spaces will be stripped. + def state_parameter_docstring(self, line: str) -> None: + if not self.valid_line(line): + return + + indent = self.indent.measure(line) + if indent < self.parameter_docstring_indent: + self.indent.infer(line) + assert self.indent.depth < 3 + if self.indent.depth == 2: + # back to a parameter + return self.next(self.state_parameter, line) + assert self.indent.depth == 1 + return self.next(self.state_function_docstring, line) + + assert self.function and self.function.parameters + last_param = next(reversed(self.function.parameters.values())) + self.docstring_append(last_param, line) + + # the final stanza of the DSL is the docstring. + def state_function_docstring(self, line: str) -> None: + assert self.function is not None + + if self.group: + fail(f"Function {self.function.name!r} has a ']' without a matching '['.") + + if not self.valid_line(line): + return + + self.docstring_append(self.function, line) + + def format_docstring_signature( + self, f: Function, parameters: list[Parameter] + ) -> str: + lines = [] + lines.append(f.displayname) + if self.forced_text_signature: + lines.append(self.forced_text_signature) + elif f.kind in {GETTER, SETTER}: + # @getter and @setter do not need signatures like a method or a function. + return '' + else: + lines.append('(') + + # populate "right_bracket_count" field for every parameter + assert parameters, "We should always have a self parameter. " + repr(f) + assert isinstance(parameters[0].converter, self_converter) + # self is always positional-only. + assert parameters[0].is_positional_only() + assert parameters[0].right_bracket_count == 0 + positional_only = True + for p in parameters[1:]: + if not p.is_positional_only(): + positional_only = False + else: + assert positional_only + if positional_only: + p.right_bracket_count = abs(p.group) + else: + # don't put any right brackets around non-positional-only parameters, ever. + p.right_bracket_count = 0 + + right_bracket_count = 0 + + def fix_right_bracket_count(desired: int) -> str: + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + need_slash = False + added_slash = False + need_a_trailing_slash = False + + # we only need a trailing slash: + # * if this is not a "docstring_only" signature + # * and if the last *shown* parameter is + # positional only + if not f.docstring_only: + for p in reversed(parameters): + if not p.converter.show_in_signature: + continue + if p.is_positional_only(): + need_a_trailing_slash = True + break + + + added_star = False + + first_parameter = True + last_p = parameters[-1] + line_length = len(''.join(lines)) + indent = " " * line_length + def add_parameter(text: str) -> None: + nonlocal line_length + nonlocal first_parameter + if first_parameter: + s = text + first_parameter = False + else: + s = ' ' + text + if line_length + len(s) >= 72: + lines.extend(["\n", indent]) + line_length = len(indent) + s = text + line_length += len(s) + lines.append(s) + + for p in parameters: + if not p.converter.show_in_signature: + continue + assert p.name + + is_self = isinstance(p.converter, self_converter) + if is_self and f.docstring_only: + # this isn't a real machine-parsable signature, + # so let's not print the "self" parameter + continue + + if p.is_positional_only(): + need_slash = not f.docstring_only + elif need_slash and not (added_slash or p.is_positional_only()): + added_slash = True + add_parameter('/,') + + if p.is_keyword_only() and not added_star: + added_star = True + add_parameter('*,') + + p_lines = [fix_right_bracket_count(p.right_bracket_count)] + + if isinstance(p.converter, self_converter): + # annotate first parameter as being a "self". + # + # if inspect.Signature gets this function, + # and it's already bound, the self parameter + # will be stripped off. + # + # if it's not bound, it should be marked + # as positional-only. + # + # note: we don't print "self" for __init__, + # because this isn't actually the signature + # for __init__. (it can't be, __init__ doesn't + # have a docstring.) if this is an __init__ + # (or __new__), then this signature is for + # calling the class to construct a new instance. + p_lines.append('$') + + if p.is_vararg(): + p_lines.append("*") + + name = p.converter.signature_name or p.name + p_lines.append(name) + + if not p.is_vararg() and p.converter.is_optional(): + p_lines.append('=') + value = p.converter.py_default + if not value: + value = repr(p.converter.default) + p_lines.append(value) + + if (p != last_p) or need_a_trailing_slash: + p_lines.append(',') + + p_output = "".join(p_lines) + add_parameter(p_output) + + lines.append(fix_right_bracket_count(0)) + if need_a_trailing_slash: + add_parameter('/') + lines.append(')') + + # PEP 8 says: + # + # The Python standard library will not use function annotations + # as that would result in a premature commitment to a particular + # annotation style. Instead, the annotations are left for users + # to discover and experiment with useful annotation styles. + # + # therefore this is commented out: + # + # if f.return_converter.py_default: + # lines.append(' -> ') + # lines.append(f.return_converter.py_default) + + if not f.docstring_only: + lines.append("\n" + libclinic.SIG_END_MARKER + "\n") + + signature_line = "".join(lines) + + # now fix up the places where the brackets look wrong + return signature_line.replace(', ]', ',] ') + + @staticmethod + def format_docstring_parameters(params: list[Parameter]) -> str: + """Create substitution text for {parameters}""" + return "".join(p.render_docstring() + "\n" for p in params if p.docstring) + + def format_docstring(self) -> str: + assert self.function is not None + f = self.function + # For the following special cases, it does not make sense to render a docstring. + if f.kind in {METHOD_INIT, METHOD_NEW, GETTER, SETTER} and not f.docstring: + return f.docstring + + # Enforce the summary line! + # The first line of a docstring should be a summary of the function. + # It should fit on one line (80 columns? 79 maybe?) and be a paragraph + # by itself. + # + # Argument Clinic enforces the following rule: + # * either the docstring is empty, + # * or it must have a summary line. + # + # Guido said Clinic should enforce this: + # http://mail.python.org/pipermail/python-dev/2013-June/127110.html + + lines = f.docstring.split('\n') + if len(lines) >= 2: + if lines[1]: + fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" + "Every non-blank function docstring must start with " + "a single line summary followed by an empty line.") + elif len(lines) == 1: + # the docstring is only one line right now--the summary line. + # add an empty line after the summary line so we have space + # between it and the {parameters} we're about to add. + lines.append('') + + parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 + if parameters_marker_count > 1: + fail('You may not specify {parameters} more than once in a docstring!') + + # insert signature at front and params after the summary line + if not parameters_marker_count: + lines.insert(2, '{parameters}') + lines.insert(0, '{signature}') + + # finalize docstring + params = f.render_parameters + parameters = self.format_docstring_parameters(params) + signature = self.format_docstring_signature(f, params) + docstring = "\n".join(lines) + return libclinic.linear_format(docstring, + signature=signature, + parameters=parameters).rstrip() + + def check_remaining_star(self, lineno: int | None = None) -> None: + assert isinstance(self.function, Function) + + if self.keyword_only: + symbol = '*' + elif self.deprecated_positional: + symbol = '* [from ...]' + else: + return + + for p in reversed(self.function.parameters.values()): + if self.keyword_only: + if p.kind == inspect.Parameter.KEYWORD_ONLY: + return + elif self.deprecated_positional: + if p.deprecated_positional == self.deprecated_positional: + return + break + + fail(f"Function {self.function.name!r} specifies {symbol!r} " + f"without following parameters.", line_number=lineno) + + def check_previous_star(self, lineno: int | None = None) -> None: + assert isinstance(self.function, Function) + + for p in self.function.parameters.values(): + if p.kind == inspect.Parameter.VAR_POSITIONAL: + fail(f"Function {self.function.name!r} uses '*' more than once.") + + + def do_post_block_processing_cleanup(self, lineno: int) -> None: + """ + Called when processing the block is done. + """ + if not self.function: + return + + self.check_remaining_star(lineno) + try: + self.function.docstring = self.format_docstring() + except ClinicError as exc: + exc.lineno = lineno + exc.filename = self.clinic.filename + raise diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index 1beed13b437886..1563fdf9065b7e 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -7,10 +7,10 @@ from collections.abc import Iterable, Iterator, Sequence from typing import Final, Any, TYPE_CHECKING if TYPE_CHECKING: - from clinic import Clinic from libclinic.converter import CConverter from libclinic.converters import self_converter from libclinic.return_converters import CReturnConverter + from libclinic.app import Clinic from libclinic import VersionTuple, unspecified diff --git a/Tools/clinic/libclinic/language.py b/Tools/clinic/libclinic/language.py index a90a9bb24e2201..15975430c16022 100644 --- a/Tools/clinic/libclinic/language.py +++ b/Tools/clinic/libclinic/language.py @@ -11,7 +11,7 @@ Module, Class, Function) if typing.TYPE_CHECKING: - from clinic import Clinic + from libclinic.app import Clinic class Language(metaclass=abc.ABCMeta): diff --git a/Tools/clinic/libclinic/parser.py b/Tools/clinic/libclinic/parser.py new file mode 100644 index 00000000000000..8135dd9ceb3937 --- /dev/null +++ b/Tools/clinic/libclinic/parser.py @@ -0,0 +1,53 @@ +from __future__ import annotations +import contextlib +import functools +import io +from types import NoneType +from typing import Any, Protocol, TYPE_CHECKING + +from libclinic import unspecified +from libclinic.block_parser import Block +from libclinic.converter import CConverter, converters +from libclinic.converters import buffer, robuffer, rwbuffer +from libclinic.return_converters import CReturnConverter, return_converters +if TYPE_CHECKING: + from libclinic.app import Clinic + + +class Parser(Protocol): + def __init__(self, clinic: Clinic) -> None: ... + def parse(self, block: Block) -> None: ... + + +@functools.cache +def _create_parser_base_namespace() -> dict[str, Any]: + ns = dict( + CConverter=CConverter, + CReturnConverter=CReturnConverter, + buffer=buffer, + robuffer=robuffer, + rwbuffer=rwbuffer, + unspecified=unspecified, + NoneType=NoneType, + ) + for name, converter in converters.items(): + ns[f'{name}_converter'] = converter + for name, return_converter in return_converters.items(): + ns[f'{name}_return_converter'] = return_converter + return ns + + +def create_parser_namespace() -> dict[str, Any]: + base_namespace = _create_parser_base_namespace() + return base_namespace.copy() + + +class PythonParser: + def __init__(self, clinic: Clinic) -> None: + pass + + def parse(self, block: Block) -> None: + namespace = create_parser_namespace() + with contextlib.redirect_stdout(io.StringIO()) as s: + exec(block.input, namespace) + block.output = s.getvalue() From 3f5bcc86d0764b691087e8412941e947554c93fd Mon Sep 17 00:00:00 2001 From: Tony Mountifield Date: Thu, 4 Apr 2024 11:32:53 +0100 Subject: [PATCH 078/143] gh-117467: Add preserving of mailbox owner on flush (GH-117510) Co-authored-by: Serhiy Storchaka --- Lib/mailbox.py | 10 +++-- Lib/test/test_mailbox.py | 42 +++++++++++++++++++ ...-04-03-18-36-53.gh-issue-117467.l6rWlj.rst | 2 + 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-03-18-36-53.gh-issue-117467.l6rWlj.rst diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 746811bd559412..b00d9e8634c785 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -750,9 +750,13 @@ def flush(self): _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() - # Make sure the new file's mode is the same as the old file's - mode = os.stat(self._path).st_mode - os.chmod(new_file.name, mode) + # Make sure the new file's mode and owner are the same as the old file's + info = os.stat(self._path) + os.chmod(new_file.name, info.st_mode) + try: + os.chown(new_file.name, info.st_uid, info.st_gid) + except (AttributeError, OSError): + pass try: os.rename(new_file.name, self._path) except FileExistsError: diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index bb24d5db1f9ed4..a1d72aed9d8939 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -9,6 +9,7 @@ import io import tempfile from test import support +from test.support import import_helper from test.support import os_helper from test.support import refleak_helper from test.support import socket_helper @@ -1081,6 +1082,47 @@ def test_permissions_after_flush(self): self.assertEqual(os.stat(self._path).st_mode, mode) + @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') + def test_ownership_after_flush(self): + # See issue gh-117467 + + pwd = import_helper.import_module('pwd') + grp = import_helper.import_module('grp') + st = os.stat(self._path) + + for e in pwd.getpwall(): + if e.pw_uid != st.st_uid: + other_uid = e.pw_uid + break + else: + self.skipTest("test needs more than one user") + + for e in grp.getgrall(): + if e.gr_gid != st.st_gid: + other_gid = e.gr_gid + break + else: + self.skipTest("test needs more than one group") + + try: + os.chown(self._path, other_uid, other_gid) + except OSError: + self.skipTest('test needs root privilege') + # Change permissions as in test_permissions_after_flush. + mode = st.st_mode | 0o666 + os.chmod(self._path, mode) + + self._box.add(self._template % 0) + i = self._box.add(self._template % 1) + # Need to remove one message to make flush() create a new file + self._box.remove(i) + self._box.flush() + + st = os.stat(self._path) + self.assertEqual(st.st_uid, other_uid) + self.assertEqual(st.st_gid, other_gid) + self.assertEqual(st.st_mode, mode) + class _TestMboxMMDF(_TestSingleFile): diff --git a/Misc/NEWS.d/next/Library/2024-04-03-18-36-53.gh-issue-117467.l6rWlj.rst b/Misc/NEWS.d/next/Library/2024-04-03-18-36-53.gh-issue-117467.l6rWlj.rst new file mode 100644 index 00000000000000..64ae9ff7b2f0b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-03-18-36-53.gh-issue-117467.l6rWlj.rst @@ -0,0 +1,2 @@ +Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush`. +Patch by Tony Mountifield. From b32789ccb91bbe43e88193f68b1364a8da6d9866 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 Apr 2024 06:39:16 -0400 Subject: [PATCH 079/143] gh-117521: Improve typing.TypeGuard docstring (#117522) --- Lib/typing.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index ef532f6c91539d..d8e4ee3635994c 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -841,22 +841,25 @@ def TypeGuard(self, parameters): 2. If the return value is ``True``, the type of its argument is the type inside ``TypeGuard``. - For example:: + For example:: + + def is_str_list(val: list[object]) -> TypeGuard[list[str]]: + '''Determines whether all objects in the list are strings''' + return all(isinstance(x, str) for x in val) - def is_str(val: Union[str, float]): - # "isinstance" type guard - if isinstance(val, str): - # Type of ``val`` is narrowed to ``str`` - ... - else: - # Else, type of ``val`` is narrowed to ``float``. - ... + def func1(val: list[object]): + if is_str_list(val): + # Type of ``val`` is narrowed to ``list[str]``. + print(" ".join(val)) + else: + # Type of ``val`` remains as ``list[object]``. + print("Not a list of strings!") Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower form of ``TypeA`` (it can even be a wider form) and this may lead to type-unsafe results. The main reason is to allow for things like - narrowing ``List[object]`` to ``List[str]`` even though the latter is not - a subtype of the former, since ``List`` is invariant. The responsibility of + narrowing ``list[object]`` to ``list[str]`` even though the latter is not + a subtype of the former, since ``list`` is invariant. The responsibility of writing type-safe type guards is left to the user. ``TypeGuard`` also works with type variables. For more information, see From df912c913a3d94995b379f1e19fe0a79acab6169 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 4 Apr 2024 13:14:44 +0100 Subject: [PATCH 080/143] [doc] Update logging documentation to improve grammar and elucidate an example. (GH-117541) --- Doc/howto/logging-cookbook.rst | 68 ++++++++++++++++++++++++++++++++-- Doc/library/logging.rst | 15 ++++---- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index d8ebeabcd522b1..61723bc6cf256a 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1915,7 +1915,7 @@ In a similar way to the above section, we can implement a listener and handler using `pynng `_, which is a Python binding to `NNG `_, billed as a spiritual successor to ZeroMQ. The following snippets illustrate -- you can test them in an environment which has -``pynng`` installed. Juat for variety, we present the listener first. +``pynng`` installed. Just for variety, we present the listener first. Subclass ``QueueListener`` @@ -1923,6 +1923,7 @@ Subclass ``QueueListener`` .. code-block:: python + # listener.py import json import logging import logging.handlers @@ -1955,7 +1956,7 @@ Subclass ``QueueListener`` break except pynng.Timeout: pass - except pynng.Closed: # sometimes hit when you hit Ctrl-C + except pynng.Closed: # sometimes happens when you hit Ctrl-C break if data is None: return None @@ -1988,6 +1989,7 @@ Subclass ``QueueHandler`` .. code-block:: python + # sender.py import json import logging import logging.handlers @@ -2015,9 +2017,10 @@ Subclass ``QueueHandler`` logging.getLogger('pynng').propagate = False handler = NNGSocketHandler(DEFAULT_ADDR) + # Make sure the process ID is in the output logging.basicConfig(level=logging.DEBUG, handlers=[logging.StreamHandler(), handler], - format='%(levelname)-8s %(name)10s %(message)s') + format='%(levelname)-8s %(name)10s %(process)6s %(message)s') levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL) logger_names = ('myapp', 'myapp.lib1', 'myapp.lib2') @@ -2031,7 +2034,64 @@ Subclass ``QueueHandler`` delay = random.random() * 2 + 0.5 time.sleep(delay) -You can run the above two snippets in separate command shells. +You can run the above two snippets in separate command shells. If we run the +listener in one shell and run the sender in two separate shells, we should see +something like the following. In the first sender shell: + +.. code-block:: console + + $ python sender.py + DEBUG myapp 613 Message no. 1 + WARNING myapp.lib2 613 Message no. 2 + CRITICAL myapp.lib2 613 Message no. 3 + WARNING myapp.lib2 613 Message no. 4 + CRITICAL myapp.lib1 613 Message no. 5 + DEBUG myapp 613 Message no. 6 + CRITICAL myapp.lib1 613 Message no. 7 + INFO myapp.lib1 613 Message no. 8 + (and so on) + +In the second sender shell: + +.. code-block:: console + + $ python sender.py + INFO myapp.lib2 657 Message no. 1 + CRITICAL myapp.lib2 657 Message no. 2 + CRITICAL myapp 657 Message no. 3 + CRITICAL myapp.lib1 657 Message no. 4 + INFO myapp.lib1 657 Message no. 5 + WARNING myapp.lib2 657 Message no. 6 + CRITICAL myapp 657 Message no. 7 + DEBUG myapp.lib1 657 Message no. 8 + (and so on) + +In the listener shell: + +.. code-block:: console + + $ python listener.py + Press Ctrl-C to stop. + DEBUG myapp 613 Message no. 1 + WARNING myapp.lib2 613 Message no. 2 + INFO myapp.lib2 657 Message no. 1 + CRITICAL myapp.lib2 613 Message no. 3 + CRITICAL myapp.lib2 657 Message no. 2 + CRITICAL myapp 657 Message no. 3 + WARNING myapp.lib2 613 Message no. 4 + CRITICAL myapp.lib1 613 Message no. 5 + CRITICAL myapp.lib1 657 Message no. 4 + INFO myapp.lib1 657 Message no. 5 + DEBUG myapp 613 Message no. 6 + WARNING myapp.lib2 657 Message no. 6 + CRITICAL myapp 657 Message no. 7 + CRITICAL myapp.lib1 613 Message no. 7 + INFO myapp.lib1 613 Message no. 8 + DEBUG myapp.lib1 657 Message no. 8 + (and so on) + +As you can see, the logging from the two sender processes is interleaved in the +listener's output. An example dictionary-based configuration diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 4e7d18bda8bf7d..7816cc20945fa8 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -63,12 +63,13 @@ If you run *myapp.py*, you should see this in *myapp.log*: INFO:mylib:Doing something INFO:__main__:Finished -The key features of this idiomatic usage is that the majority of code is simply +The key feature of this idiomatic usage is that the majority of code is simply creating a module level logger with ``getLogger(__name__)``, and using that -logger to do any needed logging. This is concise while allowing downstream code -fine grained control if needed. Logged messages to the module-level logger get -forwarded up to handlers of loggers in higher-level modules, all the way up to -the root logger; for this reason this approach is known as hierarchical logging. +logger to do any needed logging. This is concise, while allowing downstream +code fine-grained control if needed. Logged messages to the module-level logger +get forwarded to handlers of loggers in higher-level modules, all the way up to +the highest-level logger known as the root logger; this approach is known as +hierarchical logging. For logging to be useful, it needs to be configured: setting the levels and destinations for each logger, potentially changing how specific modules log, @@ -82,8 +83,8 @@ The module provides a lot of functionality and flexibility. If you are unfamiliar with logging, the best way to get to grips with it is to view the tutorials (**see the links above and on the right**). -The basic classes defined by the module, together with their functions, are -listed below. +The basic classes defined by the module, together with their attributes and +methods, are listed in the sections below. * Loggers expose the interface that application code directly uses. * Handlers send the log records (created by loggers) to the appropriate From 63bbe77d9bb2be4db83ed09b96dd22f2a44ef55b Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Thu, 4 Apr 2024 17:54:17 +0300 Subject: [PATCH 081/143] gh-109802: Add coverage test for complex_abs() (GH-117449) * gh-109802: Add coverage test for complex_abs() This tests overflow on L594. // line numbers wrt to 0f2fa6150b --- Lib/test/test_complex.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index b057121f285dc7..fa3017b24e16c8 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -10,6 +10,7 @@ INF = float("inf") NAN = float("nan") +DBL_MAX = sys.float_info.max # These tests ensure that complex math does the right thing ZERO_DIVISION = ( @@ -597,6 +598,8 @@ def test_abs(self): for num in nums: self.assertAlmostEqual((num.real**2 + num.imag**2) ** 0.5, abs(num)) + self.assertRaises(OverflowError, abs, complex(DBL_MAX, DBL_MAX)) + def test_repr_str(self): def test(v, expected, test_fn=self.assertEqual): test_fn(repr(v), expected) From 060a96f1a9a901b01ed304aa82b886d248ca1cb6 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 4 Apr 2024 08:03:27 -0700 Subject: [PATCH 082/143] gh-116968: Reimplement Tier 2 counters (#117144) Introduce a unified 16-bit backoff counter type (``_Py_BackoffCounter``), shared between the Tier 1 adaptive specializer and the Tier 2 optimizer. The API used for adaptive specialization counters is changed but the behavior is (supposed to be) identical. The behavior of the Tier 2 counters is changed: - There are no longer dynamic thresholds (we never varied these). - All counters now use the same exponential backoff. - The counter for ``JUMP_BACKWARD`` starts counting down from 16. - The ``temperature`` in side exits starts counting down from 64. --- Include/cpython/code.h | 11 ++ Include/cpython/optimizer.h | 15 +- Include/internal/pycore_backoff.h | 128 ++++++++++++++++++ Include/internal/pycore_code.h | 64 ++++----- Include/internal/pycore_interp.h | 6 - Lib/test/test_capi/test_opt.py | 16 ++- Makefile.pre.in | 1 + ...-04-03-13-44-04.gh-issue-116968.zgcdG2.rst | 11 ++ Modules/_testinternalcapi.c | 6 + PCbuild/pythoncore.vcxproj | 1 + Python/bytecodes.c | 101 ++++++-------- Python/ceval.c | 5 +- Python/ceval_macros.h | 31 ++--- Python/executor_cases.c.h | 11 +- Python/generated_cases.c.h | 87 +++++------- Python/instrumentation.c | 8 +- Python/optimizer.c | 37 +---- Python/specialize.c | 8 +- Tools/jit/template.c | 1 + 19 files changed, 313 insertions(+), 235 deletions(-) create mode 100644 Include/internal/pycore_backoff.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-03-13-44-04.gh-issue-116968.zgcdG2.rst diff --git a/Include/cpython/code.h b/Include/cpython/code.h index d5dac1765638f9..b0e226e0e1971a 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -24,6 +24,16 @@ typedef struct _Py_GlobalMonitors { uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_GlobalMonitors; +typedef struct { + union { + struct { + uint16_t backoff : 4; + uint16_t value : 12; + }; + uint16_t as_counter; // For printf("%#x", ...) + }; +} _Py_BackoffCounter; + /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG * opcode allows for larger values but the current limit is 3 uses @@ -39,6 +49,7 @@ typedef union { uint8_t code; uint8_t arg; } op; + _Py_BackoffCounter counter; // First cache entry of specializable op } _Py_CODEUNIT; diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index bc960c583782c5..819251a25bb242 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -89,7 +89,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) typedef struct _exit_data { uint32_t target; - int16_t temperature; + _Py_BackoffCounter temperature; const struct _PyExecutorObject *executor; } _PyExitData; @@ -115,11 +115,6 @@ typedef int (*optimize_func)( struct _PyOptimizerObject { PyObject_HEAD optimize_func optimize; - /* These thresholds are treated as signed so do not exceed INT16_MAX - * Use INT16_MAX to indicate that the optimizer should never be called */ - uint16_t resume_threshold; - uint16_t side_threshold; - uint16_t backedge_threshold; /* Data needed by the optimizer goes here, but is opaque to the VM */ }; @@ -151,14 +146,6 @@ extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_inval PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); -#define OPTIMIZER_BITS_IN_COUNTER 4 -/* Minimum of 16 additional executions before retry */ -#define MIN_TIER2_BACKOFF 4 -#define MAX_TIER2_BACKOFF (15 - OPTIMIZER_BITS_IN_COUNTER) -#define OPTIMIZER_BITS_MASK ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1) -/* A value <= UINT16_MAX but large enough that when shifted is > UINT16_MAX */ -#define OPTIMIZER_UNREACHABLE_THRESHOLD UINT16_MAX - #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h new file mode 100644 index 00000000000000..5d93c889e84976 --- /dev/null +++ b/Include/internal/pycore_backoff.h @@ -0,0 +1,128 @@ + +#ifndef Py_INTERNAL_BACKOFF_H +#define Py_INTERNAL_BACKOFF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include +#include + +/* 16-bit countdown counters using exponential backoff. + + These are used by the adaptive specializer to count down until + it is time to specialize an instruction. If specialization fails + the counter is reset using exponential backoff. + + Another use is for the Tier 2 optimizer to decide when to create + a new Tier 2 trace (executor). Again, exponential backoff is used. + + The 16-bit counter is structured as a 12-bit unsigned 'value' + and a 4-bit 'backoff' field. When resetting the counter, the + backoff field is incremented (until it reaches a limit) and the + value is set to a bit mask representing the value 2**backoff - 1. + The maximum backoff is 12 (the number of value bits). + + There is an exceptional value which must not be updated, 0xFFFF. +*/ + +#define UNREACHABLE_BACKOFF 0xFFFF + +static inline bool +is_unreachable_backoff_counter(_Py_BackoffCounter counter) +{ + return counter.as_counter == UNREACHABLE_BACKOFF; +} + +static inline _Py_BackoffCounter +make_backoff_counter(uint16_t value, uint16_t backoff) +{ + assert(backoff <= 15); + assert(value <= 0xFFF); + return (_Py_BackoffCounter){.value = value, .backoff = backoff}; +} + +static inline _Py_BackoffCounter +forge_backoff_counter(uint16_t counter) +{ + return (_Py_BackoffCounter){.as_counter = counter}; +} + +static inline _Py_BackoffCounter +restart_backoff_counter(_Py_BackoffCounter counter) +{ + assert(!is_unreachable_backoff_counter(counter)); + if (counter.backoff < 12) { + return make_backoff_counter((1 << (counter.backoff + 1)) - 1, counter.backoff + 1); + } + else { + return make_backoff_counter((1 << 12) - 1, 12); + } +} + +static inline _Py_BackoffCounter +pause_backoff_counter(_Py_BackoffCounter counter) +{ + return make_backoff_counter(counter.value | 1, counter.backoff); +} + +static inline _Py_BackoffCounter +advance_backoff_counter(_Py_BackoffCounter counter) +{ + if (!is_unreachable_backoff_counter(counter)) { + return make_backoff_counter((counter.value - 1) & 0xFFF, counter.backoff); + } + else { + return counter; + } +} + +static inline bool +backoff_counter_triggers(_Py_BackoffCounter counter) +{ + return counter.value == 0; +} + +/* Initial JUMP_BACKWARD counter. + * This determines when we create a trace for a loop. +* Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */ +#define JUMP_BACKWARD_INITIAL_VALUE 16 +#define JUMP_BACKWARD_INITIAL_BACKOFF 4 +static inline _Py_BackoffCounter +initial_jump_backoff_counter(void) +{ + return make_backoff_counter(JUMP_BACKWARD_INITIAL_VALUE, + JUMP_BACKWARD_INITIAL_BACKOFF); +} + +/* Initial exit temperature. + * Must be larger than ADAPTIVE_COOLDOWN_VALUE, + * otherwise when a side exit warms up we may construct + * a new trace before the Tier 1 code has properly re-specialized. + * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */ +#define COLD_EXIT_INITIAL_VALUE 64 +#define COLD_EXIT_INITIAL_BACKOFF 6 + +static inline _Py_BackoffCounter +initial_temperature_backoff_counter(void) +{ + return make_backoff_counter(COLD_EXIT_INITIAL_VALUE, + COLD_EXIT_INITIAL_BACKOFF); +} + +/* Unreachable backoff counter. */ +static inline _Py_BackoffCounter +initial_unreachable_backoff_counter(void) +{ + return forge_backoff_counter(UNREACHABLE_BACKOFF); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_BACKOFF_H */ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 6c90c9e284103c..688051bbff7aac 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -31,7 +31,7 @@ extern "C" { #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT)) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t module_keys_version; uint16_t builtin_keys_version; uint16_t index; @@ -40,44 +40,44 @@ typedef struct { #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyBinaryOpCache; #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyUnpackSequenceCache; #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \ CACHE_ENTRIES(_PyUnpackSequenceCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyCompareOpCache; #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyBinarySubscrCache; #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PySuperAttrCache; #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t version[2]; uint16_t index; } _PyAttrCache; typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t type_version[2]; union { uint16_t keys_version[2]; @@ -93,39 +93,39 @@ typedef struct { #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t func_version[2]; } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyStoreSubscrCache; #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyForIterCache; #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PySendCache; #define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t version[2]; } _PyToBoolCache; #define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyContainsOpCache; #define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache) @@ -451,18 +451,14 @@ write_location_entry_start(uint8_t *ptr, int code, int length) /** Counters * The first 16-bit value in each inline cache is a counter. - * When counting misses, the counter is treated as a simple unsigned value. * * When counting executions until the next specialization attempt, * exponential backoff is used to reduce the number of specialization failures. - * The high 12 bits store the counter, the low 4 bits store the backoff exponent. - * On a specialization failure, the backoff exponent is incremented and the - * counter set to (2**backoff - 1). - * Backoff == 6 -> starting counter == 63, backoff == 10 -> starting counter == 1023. + * See pycore_backoff.h for more details. + * On a specialization failure, the backoff counter is restarted. */ -/* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */ -#define ADAPTIVE_BACKOFF_BITS 4 +#include "pycore_backoff.h" // A value of 1 means that we attempt to specialize the *second* time each // instruction is executed. Executing twice is a much better indicator of @@ -480,36 +476,30 @@ write_location_entry_start(uint8_t *ptr, int code, int length) #define ADAPTIVE_COOLDOWN_VALUE 52 #define ADAPTIVE_COOLDOWN_BACKOFF 0 -#define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS) +// Can't assert this in pycore_backoff.h because of header order dependencies +static_assert(COLD_EXIT_INITIAL_VALUE > ADAPTIVE_COOLDOWN_VALUE, + "Cold exit value should be larger than adaptive cooldown value"); - -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_bits(uint16_t value, uint16_t backoff) { - return ((value << ADAPTIVE_BACKOFF_BITS) - | (backoff & ((1 << ADAPTIVE_BACKOFF_BITS) - 1))); + return make_backoff_counter(value, backoff); } -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_warmup(void) { return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE, ADAPTIVE_WARMUP_BACKOFF); } -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_cooldown(void) { return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE, ADAPTIVE_COOLDOWN_BACKOFF); } -static inline uint16_t -adaptive_counter_backoff(uint16_t counter) { - uint16_t backoff = counter & ((1 << ADAPTIVE_BACKOFF_BITS) - 1); - backoff++; - if (backoff > MAX_BACKOFF_VALUE) { - backoff = MAX_BACKOFF_VALUE; - } - uint16_t value = (uint16_t)(1 << backoff) - 1; - return adaptive_counter_bits(value, backoff); +static inline _Py_BackoffCounter +adaptive_counter_backoff(_Py_BackoffCounter counter) { + return restart_backoff_counter(counter); } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index b8d0fdcce11ba8..b5cea863ff35dc 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -239,12 +239,6 @@ struct _is { _PyOptimizerObject *optimizer; _PyExecutorObject *executor_list_head; - /* These two values are shifted and offset to speed up check in JUMP_BACKWARD */ - uint32_t optimizer_resume_threshold; - uint32_t optimizer_backedge_threshold; - - uint16_t optimizer_side_threshold; - _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index ceb49c3c7129cb..7ca0f6927fe4a1 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -10,6 +10,7 @@ from test.support import script_helper, requires_specialization +from _testinternalcapi import TIER2_THRESHOLD @contextlib.contextmanager def temporary_optimizer(opt): @@ -69,7 +70,8 @@ def loop(): self.assertEqual(opt.get_count(), 0) with clear_executors(loop): loop() - self.assertEqual(opt.get_count(), 1000) + # Subtract because optimizer doesn't kick in sooner + self.assertEqual(opt.get_count(), 1000 - TIER2_THRESHOLD) def test_long_loop(self): "Check that we aren't confused by EXTENDED_ARG" @@ -81,7 +83,7 @@ def nop(): pass def long_loop(): - for _ in range(10): + for _ in range(20): nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); @@ -96,7 +98,7 @@ def long_loop(): with temporary_optimizer(opt): self.assertEqual(opt.get_count(), 0) long_loop() - self.assertEqual(opt.get_count(), 10) + self.assertEqual(opt.get_count(), 20 - TIER2_THRESHOLD) # Need iterations to warm up def test_code_restore_for_ENTER_EXECUTOR(self): def testfunc(x): @@ -932,10 +934,10 @@ def testfunc(n): exec(src, ns, ns) testfunc = ns['testfunc'] ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, 16) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNone(ex) ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, 16) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_GUARD_BOTH_INT", uops) @@ -946,10 +948,10 @@ def testfunc(n): exec(src, ns, ns) testfunc = ns['testfunc'] ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, 16) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNone(ex) ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, 16) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNone(ex) def test_combine_stack_space_checks_sequential(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index 2a22a1e95a39a2..84058acdcc35fc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1124,6 +1124,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_ast.h \ $(srcdir)/Include/internal/pycore_ast_state.h \ $(srcdir)/Include/internal/pycore_atexit.h \ + $(srcdir)/Include/internal/pycore_backoff.h \ $(srcdir)/Include/internal/pycore_bitutils.h \ $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ $(srcdir)/Include/internal/pycore_brc.h \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-03-13-44-04.gh-issue-116968.zgcdG2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-03-13-44-04.gh-issue-116968.zgcdG2.rst new file mode 100644 index 00000000000000..dc5beee0022181 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-03-13-44-04.gh-issue-116968.zgcdG2.rst @@ -0,0 +1,11 @@ +Introduce a unified 16-bit backoff counter type (``_Py_BackoffCounter``), +shared between the Tier 1 adaptive specializer and the Tier 2 optimizer. The +API used for adaptive specialization counters is changed but the behavior is +(supposed to be) identical. + +The behavior of the Tier 2 counters is changed: + +* There are no longer dynamic thresholds (we never varied these). +* All counters now use the same exponential backoff. +* The counter for ``JUMP_BACKWARD`` starts counting down from 16. +* The ``temperature`` in side exits starts counting down from 64. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 6b5d99f6ffac1f..758e88e288bac6 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -10,6 +10,7 @@ #undef NDEBUG #include "Python.h" +#include "pycore_backoff.h" // JUMP_BACKWARD_INITIAL_VALUE #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_ceval.h" // _PyEval_AddPendingCall() @@ -1819,6 +1820,11 @@ module_exec(PyObject *module) return 1; } + if (PyModule_Add(module, "TIER2_THRESHOLD", + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE)) < 0) { + return 1; + } + return 0; } diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 657ffd1aa4c676..827c9f074de909 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -204,6 +204,7 @@ + diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fa53c969fe361e..8af48d9a0129b6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -8,6 +8,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_backoff.h" #include "pycore_cell.h" // PyCell_GetRef() #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS @@ -326,13 +327,13 @@ dummy_func( specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -551,13 +552,13 @@ dummy_func( specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -698,13 +699,13 @@ dummy_func( specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -982,13 +983,13 @@ dummy_func( specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -1211,13 +1212,13 @@ dummy_func( specializing op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ (void)seq; (void)counter; @@ -1280,14 +1281,14 @@ dummy_func( specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -1398,14 +1399,14 @@ dummy_func( specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -1711,7 +1712,7 @@ dummy_func( inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } @@ -1723,13 +1724,13 @@ dummy_func( specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) { #if ENABLE_SPECIALIZATION int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -1836,14 +1837,14 @@ dummy_func( specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -2157,13 +2158,13 @@ dummy_func( specializing op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -2254,13 +2255,13 @@ dummy_func( specializing op(_SPECIALIZE_CONTAINS_OP, (counter/1, left, right -- left, right)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_ContainsOp(right, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(CONTAINS_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -2340,16 +2341,8 @@ dummy_func( assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); #if ENABLE_SPECIALIZATION - uint16_t counter = this_instr[1].cache; - this_instr[1].cache = counter + (1 << OPTIMIZER_BITS_IN_COUNTER); - /* We are using unsigned values, but we really want signed values, so - * do the 2s complement adjustment manually */ - uint32_t offset_counter = counter ^ (1 << 15); - uint32_t threshold = tstate->interp->optimizer_backedge_threshold; - assert((threshold & OPTIMIZER_BITS_MASK) == 0); - // Use '>=' not '>' so that the optimizer/backoff bits do not effect the result. - // Double-check that the opcode isn't instrumented or something: - if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) { + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ while (oparg > 255) { @@ -2365,17 +2358,12 @@ dummy_func( GOTO_TIER_TWO(executor); } else { - int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK; - backoff++; - if (backoff < MIN_TIER2_BACKOFF) { - backoff = MIN_TIER2_BACKOFF; - } - else if (backoff > MAX_TIER2_BACKOFF) { - backoff = MAX_TIER2_BACKOFF; - } - this_instr[1].cache = ((UINT16_MAX << OPTIMIZER_BITS_IN_COUNTER) << backoff) | backoff; + this_instr[1].counter = restart_backoff_counter(counter); } } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } #endif /* ENABLE_SPECIALIZATION */ } @@ -2535,13 +2523,13 @@ dummy_func( specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -3001,7 +2989,7 @@ dummy_func( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); ERROR_IF(err, error); - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL); } @@ -3030,13 +3018,13 @@ dummy_func( specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } @@ -3933,13 +3921,13 @@ dummy_func( specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -3965,7 +3953,7 @@ dummy_func( ERROR_IF(next_opcode < 0, error); next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -4157,21 +4145,22 @@ dummy_func( tier2 op(_COLD_EXIT, (--)) { _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; _PyExitData *exit = &previous->exits[oparg]; - exit->temperature++; PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); - } else { + } + else { int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { - int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; - exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; + exit->temperature = restart_backoff_counter(temperature); if (optimized < 0) { Py_DECREF(previous); tstate->previous_executor = Py_None; @@ -4181,7 +4170,7 @@ dummy_func( } } /* We need two references. One to store in exit->executor and - * one to keep the executor alive when executing. */ + * one to keep the executor alive when executing. */ Py_INCREF(executor); exit->executor = executor; GOTO_TIER_TWO(executor); diff --git a/Python/ceval.c b/Python/ceval.c index f3b73165e9f28b..57ae08ee3cf85a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_backoff.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_cell.h" // PyCell_GetRef() #include "pycore_ceval.h" @@ -822,7 +823,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); /* Prevent the underlying instruction from specializing * and overwriting the instrumentation. */ - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; DISPATCH_GOTO(); @@ -1099,7 +1100,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - exit_index, exit->temperature, exit->target, + exit_index, exit->temperature.as_counter, exit->target, _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); } #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 1194c11f8ba607..224cd1da7d4a0e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -262,7 +262,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { STAT_INC(opcode, miss); \ STAT_INC((INSTNAME), miss); \ /* The counter is always the first cache entry: */ \ - if (ADAPTIVE_COUNTER_IS_ZERO(next_instr->cache)) { \ + if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \ STAT_INC((INSTNAME), deopt); \ } \ } while (0) @@ -290,29 +290,28 @@ GETITEM(PyObject *v, Py_ssize_t i) { dtrace_function_entry(frame); \ } -#define ADAPTIVE_COUNTER_IS_ZERO(COUNTER) \ - (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == 0) - -#define ADAPTIVE_COUNTER_IS_MAX(COUNTER) \ - (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == ((1 << MAX_BACKOFF_VALUE) - 1)) +/* This takes a uint16_t instead of a _Py_BackoffCounter, + * because it is used directly on the cache entry in generated code, + * which is always an integral type. */ +#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ + backoff_counter_triggers(forge_backoff_counter((COUNTER))) #ifdef Py_GIL_DISABLED -#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ - do { \ - /* gh-115999 tracks progress on addressing this. */ \ +#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + /* gh-115999 tracks progress on addressing this. */ \ static_assert(0, "The specializing interpreter is not yet thread-safe"); \ } while (0); #else -#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ - do { \ - assert(!ADAPTIVE_COUNTER_IS_ZERO((COUNTER))); \ - (COUNTER) -= (1 << ADAPTIVE_BACKOFF_BITS); \ +#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = advance_backoff_counter((COUNTER)); \ } while (0); #endif -#define INCREMENT_ADAPTIVE_COUNTER(COUNTER) \ - do { \ - (COUNTER) += (1 << ADAPTIVE_BACKOFF_BITS); \ +#define PAUSE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = pause_backoff_counter((COUNTER)); \ } while (0); #define UNBOUNDLOCAL_ERROR_MSG \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 98476798fbbbdf..8c3d41b64b49a5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3694,21 +3694,22 @@ oparg = CURRENT_OPARG(); _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; _PyExitData *exit = &previous->exits[oparg]; - exit->temperature++; PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); - } else { + } + else { int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { - int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; - exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; + exit->temperature = restart_backoff_counter(temperature); if (optimized < 0) { Py_DECREF(previous); tstate->previous_executor = Py_None; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6ee794a05b51d4..0116acd5ae302f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -115,13 +115,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -432,13 +432,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _BINARY_SUBSCR @@ -760,13 +760,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ @@ -2036,13 +2036,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _COMPARE_OP @@ -2185,13 +2185,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_ContainsOp(right, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(CONTAINS_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _CONTAINS_OP @@ -2596,13 +2596,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _FOR_ITER @@ -3026,7 +3026,7 @@ tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); if (err) goto error; - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL); } @@ -3142,7 +3142,7 @@ if (next_opcode < 0) goto error; next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -3177,7 +3177,7 @@ /* Skip 1 cache entry */ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } @@ -3415,16 +3415,8 @@ assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); #if ENABLE_SPECIALIZATION - uint16_t counter = this_instr[1].cache; - this_instr[1].cache = counter + (1 << OPTIMIZER_BITS_IN_COUNTER); - /* We are using unsigned values, but we really want signed values, so - * do the 2s complement adjustment manually */ - uint32_t offset_counter = counter ^ (1 << 15); - uint32_t threshold = tstate->interp->optimizer_backedge_threshold; - assert((threshold & OPTIMIZER_BITS_MASK) == 0); - // Use '>=' not '>' so that the optimizer/backoff bits do not effect the result. - // Double-check that the opcode isn't instrumented or something: - if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) { + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ while (oparg > 255) { @@ -3440,17 +3432,12 @@ GOTO_TIER_TWO(executor); } else { - int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK; - backoff++; - if (backoff < MIN_TIER2_BACKOFF) { - backoff = MIN_TIER2_BACKOFF; - } - else if (backoff > MAX_TIER2_BACKOFF) { - backoff = MAX_TIER2_BACKOFF; - } - this_instr[1].cache = ((UINT16_MAX << OPTIMIZER_BITS_IN_COUNTER) << backoff) | backoff; + this_instr[1].counter = restart_backoff_counter(counter); } } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } #endif /* ENABLE_SPECIALIZATION */ DISPATCH(); } @@ -3543,14 +3530,14 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 8 cache entries */ @@ -4238,14 +4225,14 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 1 cache entry */ @@ -4442,13 +4429,13 @@ (void)counter; #if ENABLE_SPECIALIZATION int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _LOAD_SUPER_ATTR @@ -5083,13 +5070,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _SEND @@ -5271,14 +5258,14 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 3 cache entries */ @@ -5562,13 +5549,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _STORE_SUBSCR @@ -5665,13 +5652,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ @@ -5882,13 +5869,13 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ (void)seq; (void)counter; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 018cd662b1561a..0f60290865000c 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -590,7 +590,7 @@ de_instrument(PyCodeObject *code, int i, int event) CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); *opcode_ptr = deinstrumented; if (_PyOpcode_Caches[deinstrumented]) { - instr[1].cache = adaptive_counter_warmup(); + instr[1].counter = adaptive_counter_warmup(); } } @@ -611,7 +611,7 @@ de_instrument_line(PyCodeObject *code, int i) CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); instr->op.code = original_opcode; if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); + instr[1].counter = adaptive_counter_warmup(); } assert(instr->op.code != INSTRUMENTED_LINE); } @@ -634,7 +634,7 @@ de_instrument_per_instruction(PyCodeObject *code, int i) CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); *opcode_ptr = original_opcode; if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); + instr[1].counter = adaptive_counter_warmup(); } assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION); assert(instr->op.code != INSTRUMENTED_INSTRUCTION); @@ -667,7 +667,7 @@ instrument(PyCodeObject *code, int i) assert(instrumented); *opcode_ptr = instrumented; if (_PyOpcode_Caches[deopt]) { - instr[1].cache = adaptive_counter_warmup(); + instr[1].counter = adaptive_counter_warmup(); } } } diff --git a/Python/optimizer.c b/Python/optimizer.c index 38ab6d3cf61c72..5c69d9d5de92eb 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,6 +1,7 @@ #include "Python.h" #include "opcode.h" #include "pycore_interp.h" +#include "pycore_backoff.h" #include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_metadata.h" // _PyOpcode_OpName[] @@ -110,9 +111,7 @@ never_optimize( _PyExecutorObject **exec, int Py_UNUSED(stack_entries)) { - /* Although it should be benign for this to be called, - * it shouldn't happen, so fail in debug builds. */ - assert(0 && "never optimize should never be called"); + // This may be called if the optimizer is reset return 0; } @@ -127,25 +126,12 @@ PyTypeObject _PyDefaultOptimizer_Type = { static _PyOptimizerObject _PyOptimizer_Default = { PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type) .optimize = never_optimize, - .resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, - .backedge_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, - .side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, }; -static uint32_t -shift_and_offset_threshold(uint32_t threshold) -{ - return (threshold << OPTIMIZER_BITS_IN_COUNTER) + (1 << 15); -} - _PyOptimizerObject * PyUnstable_GetOptimizer(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(interp->optimizer_backedge_threshold == - shift_and_offset_threshold(interp->optimizer->backedge_threshold)); - assert(interp->optimizer_resume_threshold == - shift_and_offset_threshold(interp->optimizer->resume_threshold)); if (interp->optimizer == &_PyOptimizer_Default) { return NULL; } @@ -190,13 +176,6 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) } Py_INCREF(optimizer); interp->optimizer = optimizer; - interp->optimizer_backedge_threshold = shift_and_offset_threshold(optimizer->backedge_threshold); - interp->optimizer_resume_threshold = shift_and_offset_threshold(optimizer->resume_threshold); - interp->optimizer_side_threshold = optimizer->side_threshold; - if (optimizer == &_PyOptimizer_Default) { - assert(interp->optimizer_backedge_threshold > (1 << 16)); - assert(interp->optimizer_resume_threshold > (1 << 16)); - } return old; } @@ -1109,7 +1088,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil assert(exit_count < COLD_EXIT_COUNT); for (int i = 0; i < exit_count; i++) { executor->exits[i].executor = &COLD_EXITS[i]; - executor->exits[i].temperature = 0; + executor->exits[i].temperature = initial_temperature_backoff_counter(); } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; @@ -1291,11 +1270,6 @@ PyUnstable_Optimizer_NewUOpOptimizer(void) return NULL; } opt->optimize = uop_optimize; - opt->resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; - // Need a few iterations to settle specializations, - // and to ammortize the cost of optimization. - opt->side_threshold = 16; - opt->backedge_threshold = 16; return (PyObject *)opt; } @@ -1385,9 +1359,6 @@ PyUnstable_Optimizer_NewCounter(void) return NULL; } opt->base.optimize = counter_optimize; - opt->base.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; - opt->base.side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; - opt->base.backedge_threshold = 0; opt->count = 0; return (PyObject *)opt; } @@ -1554,7 +1525,7 @@ _Py_ExecutorClear(_PyExecutorObject *executor) for (uint32_t i = 0; i < executor->exit_count; i++) { Py_DECREF(executor->exits[i].executor); executor->exits[i].executor = &COLD_EXITS[i]; - executor->exits[i].temperature = INT16_MIN; + executor->exits[i].temperature = initial_unreachable_backoff_counter(); } _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; assert(instruction->op.code == ENTER_EXECUTOR); diff --git a/Python/specialize.c b/Python/specialize.c index f1e32d05af7707..0b4b199a23e297 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -419,22 +419,20 @@ _PyCode_Quicken(PyCodeObject *code) int caches = _PyOpcode_Caches[opcode]; if (caches) { // The initial value depends on the opcode - int initial_value; switch (opcode) { case JUMP_BACKWARD: - initial_value = 0; + instructions[i + 1].counter = initial_jump_backoff_counter(); break; case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: case POP_JUMP_IF_NONE: case POP_JUMP_IF_NOT_NONE: - initial_value = 0x5555; // Alternating 0, 1 bits + instructions[i + 1].cache = 0x5555; // Alternating 0, 1 bits break; default: - initial_value = adaptive_counter_warmup(); + instructions[i + 1].counter = adaptive_counter_warmup(); break; } - instructions[i + 1].cache = initial_value; i += caches; } } diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 54160084cda460..351bc2f3dd48de 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -1,5 +1,6 @@ #include "Python.h" +#include "pycore_backoff.h" #include "pycore_call.h" #include "pycore_ceval.h" #include "pycore_cell.h" From 04697bcfaf5dd34c9312f4f405083b6d33b3511f Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:47:26 +0100 Subject: [PATCH 083/143] gh-117494: extract the Instruction Sequence data structure into a separate file (#117496) --- Include/internal/pycore_compile.h | 33 +-- Include/internal/pycore_flowgraph.h | 15 +- .../internal/pycore_instruction_sequence.h | 60 ++++++ Makefile.pre.in | 12 +- ...-04-04-13-42-59.gh-issue-117494.GPQH64.rst | 1 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 1 + PCbuild/pythoncore.vcxproj | 2 + PCbuild/pythoncore.vcxproj.filters | 6 + Python/assemble.c | 12 +- Python/compile.c | 197 +++--------------- Python/flowgraph.c | 16 +- Python/instruction_sequence.c | 151 ++++++++++++++ 13 files changed, 283 insertions(+), 224 deletions(-) create mode 100644 Include/internal/pycore_instruction_sequence.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-04-13-42-59.gh-issue-117494.GPQH64.rst create mode 100644 Python/instruction_sequence.c diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index eb6e5ca58f7390..3c21f83a18b52a 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_symtable.h" // _Py_SourceLocation +#include "pycore_instruction_sequence.h" struct _arena; // Type defined in pycore_pyarena.h struct _mod; // Type defined in pycore_ast.h @@ -37,38 +38,6 @@ extern int _PyAST_Optimize( int optimize, int ff_features); -typedef struct { - int h_label; - int h_startdepth; - int h_preserve_lasti; -} _PyCompile_ExceptHandlerInfo; - -typedef struct { - int i_opcode; - int i_oparg; - _Py_SourceLocation i_loc; - _PyCompile_ExceptHandlerInfo i_except_handler_info; - - /* Used by the assembler */ - int i_target; - int i_offset; -} _PyCompile_Instruction; - -typedef struct { - _PyCompile_Instruction *s_instrs; - int s_allocated; - int s_used; - - int *s_labelmap; /* label id --> instr offset */ - int s_labelmap_size; - int s_next_free_label; /* next free label id */ -} _PyCompile_InstructionSequence; - -int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl); -int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq, - int opcode, int oparg, - _Py_SourceLocation loc); -int _PyCompile_InstructionSequence_ApplyLabelMap(_PyCompile_InstructionSequence *seq); typedef struct { PyObject *u_name; diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 121302aacb3a8b..819117b83114bc 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -8,16 +8,13 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode_utils.h" #include "pycore_compile.h" - -typedef struct { - int id; -} _PyCfgJumpTargetLabel; +#include "pycore_instruction_sequence.h" +#include "pycore_opcode_utils.h" struct _PyCfgBuilder; -int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); +int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyJumpTargetLabel lbl); int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _Py_SourceLocation loc); struct _PyCfgBuilder* _PyCfgBuilder_New(void); @@ -27,14 +24,14 @@ int _PyCfgBuilder_CheckSize(struct _PyCfgBuilder* g); int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, int nlocals, int nparams, int firstlineno); -int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_InstructionSequence *seq); +int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyInstructionSequence *seq); int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, int code_flags, int *stackdepth, int *nlocalsplus, - _PyCompile_InstructionSequence *seq); + _PyInstructionSequence *seq); PyCodeObject * _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache, - PyObject *consts, int maxdepth, _PyCompile_InstructionSequence *instrs, + PyObject *consts, int maxdepth, _PyInstructionSequence *instrs, int nlocalsplus, int code_flags, PyObject *filename); #ifdef __cplusplus diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h new file mode 100644 index 00000000000000..b57484fa05309f --- /dev/null +++ b/Include/internal/pycore_instruction_sequence.h @@ -0,0 +1,60 @@ +#ifndef Py_INTERNAL_INSTRUCTION_SEQUENCE_H +#define Py_INTERNAL_INSTRUCTION_SEQUENCE_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + int h_label; + int h_startdepth; + int h_preserve_lasti; +} _PyExceptHandlerInfo; + +typedef struct { + int i_opcode; + int i_oparg; + _Py_SourceLocation i_loc; + _PyExceptHandlerInfo i_except_handler_info; + + /* Temporary fields, used by the assembler and in instr_sequence_to_cfg */ + int i_target; + int i_offset; +} _PyInstruction; + +typedef struct { + _PyInstruction *s_instrs; + int s_allocated; + int s_used; + + int s_next_free_label; /* next free label id */ + /* Map of a label id to instruction offset (index into s_instrs). + * If s_labelmap is NULL, then each label id is the offset itself. + */ + int *s_labelmap; /* label id --> instr offset */ + int s_labelmap_size; +} _PyInstructionSequence; + +typedef struct { + int id; +} _PyJumpTargetLabel; + +int _PyInstructionSequence_UseLabel(_PyInstructionSequence *seq, int lbl); +int _PyInstructionSequence_Addop(_PyInstructionSequence *seq, + int opcode, int oparg, + _Py_SourceLocation loc); +_PyJumpTargetLabel _PyInstructionSequence_NewLabel(_PyInstructionSequence *seq); +int _PyInstructionSequence_ApplyLabelMap(_PyInstructionSequence *seq); +int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int pos, + int opcode, int oparg, _Py_SourceLocation loc); +void PyInstructionSequence_Fini(_PyInstructionSequence *seq); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_INSTRUCTION_SEQUENCE_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 84058acdcc35fc..fd8678cdaf8207 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -442,6 +442,7 @@ PYTHON_OBJS= \ Python/initconfig.o \ Python/interpconfig.o \ Python/instrumentation.o \ + Python/instruction_sequence.o \ Python/intrinsics.o \ Python/jit.o \ Python/legacy_tracing.o \ @@ -1170,6 +1171,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_importdl.h \ $(srcdir)/Include/internal/pycore_initconfig.h \ $(srcdir)/Include/internal/pycore_instruments.h \ + $(srcdir)/Include/internal/pycore_instruction_sequence.h \ $(srcdir)/Include/internal/pycore_interp.h \ $(srcdir)/Include/internal/pycore_intrinsics.h \ $(srcdir)/Include/internal/pycore_jit.h \ @@ -1800,7 +1802,7 @@ regen-sre: $(srcdir)/Modules/_sre/sre_constants.h \ $(srcdir)/Modules/_sre/sre_targets.h -Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h +Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h Python/getplatform.o: $(srcdir)/Python/getplatform.c $(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c @@ -1935,8 +1937,12 @@ regen-uop-metadata: $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new - -Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h +Python/compile.o Python/assemble.o Python/flowgraph.o Python/instruction_sequence.o: \ + $(srcdir)/Include/internal/pycore_compile.h \ + $(srcdir)/Include/internal/pycore_flowgraph.h \ + $(srcdir)/Include/internal/pycore_instruction_sequence.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_opcode_utils.h Python/ceval.o: \ $(srcdir)/Python/ceval_macros.h \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-04-13-42-59.gh-issue-117494.GPQH64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-04-13-42-59.gh-issue-117494.GPQH64.rst new file mode 100644 index 00000000000000..3b550eda64834b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-04-13-42-59.gh-issue-117494.GPQH64.rst @@ -0,0 +1 @@ +Refactored the instruction sequence data structure out of compile.c into instruction_sequence.c. diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 9c82fcf021bb55..9717d89b54d828 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -222,6 +222,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 63b033a0350b20..9b106bea601e34 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -229,6 +229,7 @@ Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 827c9f074de909..3a019a5fe550db 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -248,6 +248,7 @@ + @@ -590,6 +591,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 6e0cd1754f5cff..e43970410bd378 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -663,6 +663,9 @@ Include\internal + + Include\internal + Include\internal @@ -1349,6 +1352,9 @@ Source Files + + Source Files + Source Files diff --git a/Python/assemble.c b/Python/assemble.c index be3d9c1a74657c..945c8ac39f53ac 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_code.h" // write_location_entry_start() #include "pycore_compile.h" +#include "pycore_instruction_sequence.h" #include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE #include "pycore_opcode_metadata.h" // is_pseudo_target, _PyOpcode_Caches #include "pycore_symtable.h" // _Py_SourceLocation @@ -23,8 +24,8 @@ } typedef _Py_SourceLocation location; -typedef _PyCompile_Instruction instruction; -typedef _PyCompile_InstructionSequence instr_sequence; +typedef _PyInstruction instruction; +typedef _PyInstructionSequence instr_sequence; static inline bool same_location(location a, location b) @@ -132,7 +133,7 @@ assemble_emit_exception_table_item(struct assembler *a, int value, int msb) static int assemble_emit_exception_table_entry(struct assembler *a, int start, int end, int handler_offset, - _PyCompile_ExceptHandlerInfo *handler) + _PyExceptHandlerInfo *handler) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_except_table); if (a->a_except_table_off + MAX_SIZE_OF_ENTRY >= len) { @@ -158,7 +159,7 @@ static int assemble_exception_table(struct assembler *a, instr_sequence *instrs) { int ioffset = 0; - _PyCompile_ExceptHandlerInfo handler; + _PyExceptHandlerInfo handler; handler.h_label = -1; handler.h_startdepth = -1; handler.h_preserve_lasti = -1; @@ -736,8 +737,7 @@ _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *umd, PyObject *const_cac PyObject *consts, int maxdepth, instr_sequence *instrs, int nlocalsplus, int code_flags, PyObject *filename) { - - if (_PyCompile_InstructionSequence_ApplyLabelMap(instrs) < 0) { + if (_PyInstructionSequence_ApplyLabelMap(instrs) < 0) { return NULL; } if (resolve_unconditional_jumps(instrs) < 0) { diff --git a/Python/compile.c b/Python/compile.c index d9312f93d0680f..1e8f97e72cdff6 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -86,7 +86,7 @@ location_is_after(location loc1, location loc2) { #define LOC(x) SRC_LOCATION_FROM_AST(x) -typedef _PyCfgJumpTargetLabel jump_target_label; +typedef _PyJumpTargetLabel jump_target_label; static jump_target_label NO_LABEL = {-1}; @@ -94,13 +94,13 @@ static jump_target_label NO_LABEL = {-1}; #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) #define NEW_JUMP_TARGET_LABEL(C, NAME) \ - jump_target_label NAME = instr_sequence_new_label(INSTR_SEQUENCE(C)); \ + jump_target_label NAME = _PyInstructionSequence_NewLabel(INSTR_SEQUENCE(C)); \ if (!IS_LABEL(NAME)) { \ return ERROR; \ } #define USE_LABEL(C, LBL) \ - RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id)) + RETURN_IF_ERROR(_PyInstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id)) /* fblockinfo tracks the current frame block. @@ -134,8 +134,8 @@ enum { }; -typedef _PyCompile_Instruction instruction; -typedef _PyCompile_InstructionSequence instr_sequence; +typedef _PyInstruction instruction; +typedef _PyInstructionSequence instr_sequence; #define INITIAL_INSTR_SEQUENCE_SIZE 100 #define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 @@ -195,168 +195,35 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, return SUCCESS; } -static int -instr_sequence_next_inst(instr_sequence *seq) { - assert(seq->s_instrs != NULL || seq->s_used == 0); - - RETURN_IF_ERROR( - _PyCompile_EnsureArrayLargeEnough(seq->s_used + 1, - (void**)&seq->s_instrs, - &seq->s_allocated, - INITIAL_INSTR_SEQUENCE_SIZE, - sizeof(instruction))); - assert(seq->s_allocated >= 0); - assert(seq->s_used < seq->s_allocated); - return seq->s_used++; -} - -static jump_target_label -instr_sequence_new_label(instr_sequence *seq) -{ - jump_target_label lbl = {++seq->s_next_free_label}; - return lbl; -} - -int -_PyCompile_InstructionSequence_UseLabel(instr_sequence *seq, int lbl) -{ - int old_size = seq->s_labelmap_size; - RETURN_IF_ERROR( - _PyCompile_EnsureArrayLargeEnough(lbl, - (void**)&seq->s_labelmap, - &seq->s_labelmap_size, - INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, - sizeof(int))); - - for(int i = old_size; i < seq->s_labelmap_size; i++) { - seq->s_labelmap[i] = -111; /* something weird, for debugging */ - } - seq->s_labelmap[lbl] = seq->s_used; /* label refers to the next instruction */ - return SUCCESS; -} - -int -_PyCompile_InstructionSequence_ApplyLabelMap(instr_sequence *instrs) -{ - /* Replace labels by offsets in the code */ - for (int i=0; i < instrs->s_used; i++) { - instruction *instr = &instrs->s_instrs[i]; - if (HAS_TARGET(instr->i_opcode)) { - assert(instr->i_oparg < instrs->s_labelmap_size); - instr->i_oparg = instrs->s_labelmap[instr->i_oparg]; - } - _PyCompile_ExceptHandlerInfo *hi = &instr->i_except_handler_info; - if (hi->h_label >= 0) { - assert(hi->h_label < instrs->s_labelmap_size); - hi->h_label = instrs->s_labelmap[hi->h_label]; - } - } - /* Clear label map so it's never used again */ - PyMem_Free(instrs->s_labelmap); - instrs->s_labelmap = NULL; - instrs->s_labelmap_size = 0; - return SUCCESS; -} - -#define MAX_OPCODE 511 - -int -_PyCompile_InstructionSequence_Addop(instr_sequence *seq, int opcode, int oparg, - location loc) -{ - assert(0 <= opcode && opcode <= MAX_OPCODE); - assert(IS_WITHIN_OPCODE_RANGE(opcode)); - assert(OPCODE_HAS_ARG(opcode) || HAS_TARGET(opcode) || oparg == 0); - assert(0 <= oparg && oparg < (1 << 30)); - - int idx = instr_sequence_next_inst(seq); - RETURN_IF_ERROR(idx); - instruction *ci = &seq->s_instrs[idx]; - ci->i_opcode = opcode; - ci->i_oparg = oparg; - ci->i_loc = loc; - return SUCCESS; -} - -static int -instr_sequence_insert_instruction(instr_sequence *seq, int pos, - int opcode, int oparg, location loc) -{ - assert(pos >= 0 && pos <= seq->s_used); - int last_idx = instr_sequence_next_inst(seq); - RETURN_IF_ERROR(last_idx); - for (int i=last_idx-1; i >= pos; i--) { - seq->s_instrs[i+1] = seq->s_instrs[i]; - } - instruction *ci = &seq->s_instrs[pos]; - ci->i_opcode = opcode; - ci->i_oparg = oparg; - ci->i_loc = loc; - - /* fix the labels map */ - for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { - if (seq->s_labelmap[lbl] >= pos) { - seq->s_labelmap[lbl]++; - } - } - return SUCCESS; -} - -static void -instr_sequence_fini(instr_sequence *seq) { - PyMem_Free(seq->s_labelmap); - seq->s_labelmap = NULL; - - PyMem_Free(seq->s_instrs); - seq->s_instrs = NULL; -} - static cfg_builder* instr_sequence_to_cfg(instr_sequence *seq) { + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { + return NULL; + } cfg_builder *g = _PyCfgBuilder_New(); if (g == NULL) { return NULL; } - - /* There can be more than one label for the same offset. The - * offset2lbl maping selects one of them which we use consistently. - */ - - int *offset2lbl = PyMem_Malloc(seq->s_used * sizeof(int)); - if (offset2lbl == NULL) { - PyErr_NoMemory(); - goto error; - } for (int i = 0; i < seq->s_used; i++) { - offset2lbl[i] = -1; + seq->s_instrs[i].i_target = 0; } - for (int lbl=0; lbl < seq->s_labelmap_size; lbl++) { - int offset = seq->s_labelmap[lbl]; - if (offset >= 0) { - assert(offset < seq->s_used); - offset2lbl[offset] = lbl; + for (int i = 0; i < seq->s_used; i++) { + instruction *instr = &seq->s_instrs[i]; + if (HAS_TARGET(instr->i_opcode)) { + assert(instr->i_oparg >= 0 && instr->i_oparg < seq->s_used); + seq->s_instrs[instr->i_oparg].i_target = 1; } } - for (int i = 0; i < seq->s_used; i++) { - int lbl = offset2lbl[i]; - if (lbl >= 0) { - assert (lbl < seq->s_labelmap_size); - jump_target_label lbl_ = {lbl}; + instruction *instr = &seq->s_instrs[i]; + if (instr->i_target) { + jump_target_label lbl_ = {i}; if (_PyCfgBuilder_UseLabel(g, lbl_) < 0) { goto error; } } - instruction *instr = &seq->s_instrs[i]; int opcode = instr->i_opcode; int oparg = instr->i_oparg; - if (HAS_TARGET(opcode)) { - int offset = seq->s_labelmap[oparg]; - assert(offset >= 0 && offset < seq->s_used); - int lbl = offset2lbl[offset]; - assert(lbl >= 0 && lbl < seq->s_labelmap_size); - oparg = lbl; - } if (_PyCfgBuilder_Addop(g, opcode, oparg, instr->i_loc) < 0) { goto error; } @@ -364,11 +231,9 @@ instr_sequence_to_cfg(instr_sequence *seq) { if (_PyCfgBuilder_CheckSize(g) < 0) { goto error; } - PyMem_Free(offset2lbl); return g; error: _PyCfgBuilder_Free(g); - PyMem_Free(offset2lbl); return NULL; } @@ -702,7 +567,7 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) static void compiler_unit_free(struct compiler_unit *u) { - instr_sequence_fini(&u->u_instr_sequence); + PyInstructionSequence_Fini(&u->u_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_metadata.u_name); Py_CLEAR(u->u_metadata.u_qualname); @@ -952,7 +817,7 @@ codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { assert(!OPCODE_HAS_ARG(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyCompile_InstructionSequence_Addop(seq, opcode, 0, loc); + return _PyInstructionSequence_Addop(seq, opcode, 0, loc); } static Py_ssize_t @@ -1185,7 +1050,7 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyCompile_InstructionSequence_Addop(seq, opcode, oparg_, loc); + return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc); } static int @@ -1195,7 +1060,7 @@ codegen_addop_j(instr_sequence *seq, location loc, assert(IS_LABEL(target)); assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyCompile_InstructionSequence_Addop(seq, opcode, target.id, loc); + return _PyInstructionSequence_Addop(seq, opcode, target.id, loc); } #define RETURN_IF_ERROR_IN_SCOPE(C, CALL) { \ @@ -2217,7 +2082,7 @@ wrap_in_stopiteration_handler(struct compiler *c) /* Insert SETUP_CLEANUP at start */ RETURN_IF_ERROR( - instr_sequence_insert_instruction( + _PyInstructionSequence_InsertInstruction( INSTR_SEQUENCE(c), 0, SETUP_CLEANUP, handler.id, NO_LOCATION)); @@ -7690,7 +7555,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, error: Py_XDECREF(consts); - instr_sequence_fini(&optimized_instrs); + PyInstructionSequence_Fini(&optimized_instrs); _PyCfgBuilder_Free(g); return co; } @@ -7763,7 +7628,7 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) for (int i = 0; i < num_insts; i++) { if (is_target[i]) { - if (_PyCompile_InstructionSequence_UseLabel(seq, i) < 0) { + if (_PyInstructionSequence_UseLabel(seq, i) < 0) { goto error; } } @@ -7803,7 +7668,7 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) if (PyErr_Occurred()) { goto error; } - if (_PyCompile_InstructionSequence_Addop(seq, opcode, oparg, loc) < 0) { + if (_PyInstructionSequence_Addop(seq, opcode, oparg, loc) < 0) { goto error; } } @@ -7828,11 +7693,11 @@ instructions_to_cfg(PyObject *instructions) if (g == NULL) { goto error; } - instr_sequence_fini(&seq); + PyInstructionSequence_Fini(&seq); return g; error: _PyCfgBuilder_Free(g); - instr_sequence_fini(&seq); + PyInstructionSequence_Fini(&seq); return NULL; } @@ -7874,11 +7739,11 @@ cfg_to_instructions(cfg_builder *g) if (_PyCfg_ToInstructionSequence(g, &seq) < 0) { return NULL; } - if (_PyCompile_InstructionSequence_ApplyLabelMap(&seq) < 0) { + if (_PyInstructionSequence_ApplyLabelMap(&seq) < 0) { return NULL; } PyObject *res = instr_sequence_to_instructions(&seq); - instr_sequence_fini(&seq); + PyInstructionSequence_Fini(&seq); return res; } @@ -8048,7 +7913,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, goto finally; } - if (_PyCompile_InstructionSequence_ApplyLabelMap(INSTR_SEQUENCE(c)) < 0) { + if (_PyInstructionSequence_ApplyLabelMap(INSTR_SEQUENCE(c)) < 0) { return NULL; } @@ -8138,7 +8003,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, error: Py_DECREF(const_cache); _PyCfgBuilder_Free(g); - instr_sequence_fini(&optimized_instrs); + PyInstructionSequence_Fini(&optimized_instrs); return co; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 9d98f6910cdf54..83768023a4d870 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -23,7 +23,7 @@ #define DEFAULT_BLOCK_SIZE 16 typedef _Py_SourceLocation location; -typedef _PyCfgJumpTargetLabel jump_target_label; +typedef _PyJumpTargetLabel jump_target_label; typedef struct _PyCfgInstruction { int i_opcode; @@ -40,7 +40,7 @@ typedef struct _PyCfgBasicblock { control flow. */ struct _PyCfgBasicblock *b_list; /* The label of this block if it is a jump target, -1 otherwise */ - _PyCfgJumpTargetLabel b_label; + _PyJumpTargetLabel b_label; /* Exception stack at start of block, used by assembler to create the exception handling table */ struct _PyCfgExceptStack *b_exceptstack; /* pointer to an array of instructions, initially NULL */ @@ -81,7 +81,7 @@ struct _PyCfgBuilder { /* pointer to the block currently being constructed */ struct _PyCfgBasicblock *g_curblock; /* label for the next instruction to be placed */ - _PyCfgJumpTargetLabel g_current_label; + _PyJumpTargetLabel g_current_label; }; typedef struct _PyCfgBuilder cfg_builder; @@ -2712,7 +2712,7 @@ prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_fl } int -_PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq) +_PyCfg_ToInstructionSequence(cfg_builder *g, _PyInstructionSequence *seq) { int lbl = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { @@ -2720,7 +2720,7 @@ _PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq lbl += 1; } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(seq, b->b_label.id)); + RETURN_IF_ERROR(_PyInstructionSequence_UseLabel(seq, b->b_label.id)); for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; if (HAS_TARGET(instr->i_opcode)) { @@ -2728,10 +2728,10 @@ _PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq instr->i_oparg = instr->i_target->b_label.id; } RETURN_IF_ERROR( - _PyCompile_InstructionSequence_Addop( + _PyInstructionSequence_Addop( seq, instr->i_opcode, instr->i_oparg, instr->i_loc)); - _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; + _PyExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; if (instr->i_except != NULL) { hi->h_label = instr->i_except->b_label.id; hi->h_startdepth = instr->i_except->b_startdepth; @@ -2750,7 +2750,7 @@ int _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, _PyCompile_CodeUnitMetadata *umd, int code_flags, int *stackdepth, int *nlocalsplus, - _PyCompile_InstructionSequence *seq) + _PyInstructionSequence *seq) { *stackdepth = calculate_stackdepth(g); if (*stackdepth < 0) { diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c new file mode 100644 index 00000000000000..597d2b73d19f30 --- /dev/null +++ b/Python/instruction_sequence.c @@ -0,0 +1,151 @@ +/* + * This file implements a data structure representing a sequence of + * instructions, which is used by different parts of the compilation + * pipeline. + */ + + +#include + +#include "Python.h" + +#include "pycore_compile.h" // _PyCompile_EnsureArrayLargeEnough +#include "pycore_opcode_utils.h" +#include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc + +typedef _PyInstruction instruction; +typedef _PyInstructionSequence instr_sequence; +typedef _Py_SourceLocation location; + +#define INITIAL_INSTR_SEQUENCE_SIZE 100 +#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 + +#undef SUCCESS +#undef ERROR +#define SUCCESS 0 +#define ERROR -1 + +#define RETURN_IF_ERROR(X) \ + if ((X) == -1) { \ + return ERROR; \ + } + +static int +instr_sequence_next_inst(instr_sequence *seq) { + assert(seq->s_instrs != NULL || seq->s_used == 0); + + RETURN_IF_ERROR( + _PyCompile_EnsureArrayLargeEnough(seq->s_used + 1, + (void**)&seq->s_instrs, + &seq->s_allocated, + INITIAL_INSTR_SEQUENCE_SIZE, + sizeof(instruction))); + assert(seq->s_allocated >= 0); + assert(seq->s_used < seq->s_allocated); + return seq->s_used++; +} + +_PyJumpTargetLabel +_PyInstructionSequence_NewLabel(instr_sequence *seq) +{ + _PyJumpTargetLabel lbl = {++seq->s_next_free_label}; + return lbl; +} + +int +_PyInstructionSequence_UseLabel(instr_sequence *seq, int lbl) +{ + int old_size = seq->s_labelmap_size; + RETURN_IF_ERROR( + _PyCompile_EnsureArrayLargeEnough(lbl, + (void**)&seq->s_labelmap, + &seq->s_labelmap_size, + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, + sizeof(int))); + + for(int i = old_size; i < seq->s_labelmap_size; i++) { + seq->s_labelmap[i] = -111; /* something weird, for debugging */ + } + seq->s_labelmap[lbl] = seq->s_used; /* label refers to the next instruction */ + return SUCCESS; +} + +int +_PyInstructionSequence_ApplyLabelMap(instr_sequence *instrs) +{ + if (instrs->s_labelmap == NULL) { + /* Already applied - nothing to do */ + return SUCCESS; + } + /* Replace labels by offsets in the code */ + for (int i=0; i < instrs->s_used; i++) { + instruction *instr = &instrs->s_instrs[i]; + if (HAS_TARGET(instr->i_opcode)) { + assert(instr->i_oparg < instrs->s_labelmap_size); + instr->i_oparg = instrs->s_labelmap[instr->i_oparg]; + } + _PyExceptHandlerInfo *hi = &instr->i_except_handler_info; + if (hi->h_label >= 0) { + assert(hi->h_label < instrs->s_labelmap_size); + hi->h_label = instrs->s_labelmap[hi->h_label]; + } + } + /* Clear label map so it's never used again */ + PyMem_Free(instrs->s_labelmap); + instrs->s_labelmap = NULL; + instrs->s_labelmap_size = 0; + return SUCCESS; +} + +#define MAX_OPCODE 511 + +int +_PyInstructionSequence_Addop(instr_sequence *seq, int opcode, int oparg, + location loc) +{ + assert(0 <= opcode && opcode <= MAX_OPCODE); + assert(IS_WITHIN_OPCODE_RANGE(opcode)); + assert(OPCODE_HAS_ARG(opcode) || HAS_TARGET(opcode) || oparg == 0); + assert(0 <= oparg && oparg < (1 << 30)); + + int idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(idx); + instruction *ci = &seq->s_instrs[idx]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + return SUCCESS; +} + +int +_PyInstructionSequence_InsertInstruction(instr_sequence *seq, int pos, + int opcode, int oparg, location loc) +{ + assert(pos >= 0 && pos <= seq->s_used); + int last_idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(last_idx); + for (int i=last_idx-1; i >= pos; i--) { + seq->s_instrs[i+1] = seq->s_instrs[i]; + } + instruction *ci = &seq->s_instrs[pos]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + + /* fix the labels map */ + for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { + if (seq->s_labelmap[lbl] >= pos) { + seq->s_labelmap[lbl]++; + } + } + return SUCCESS; +} + +void +PyInstructionSequence_Fini(instr_sequence *seq) { + PyMem_Free(seq->s_labelmap); + seq->s_labelmap = NULL; + + PyMem_Free(seq->s_instrs); + seq->s_instrs = NULL; +} From de5ca0bf71760aad8f2b8449c89242498bff64c8 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 4 Apr 2024 14:09:38 -0400 Subject: [PATCH 084/143] gh-117435: Make `SemLock` thread-safe in free-threaded build (#117436) Use critical sections to make acquire, release, and _count thread-safe without the GIL. --- Modules/_multiprocessing/clinic/semaphore.c.h | 31 ++++++++++++++++--- Modules/_multiprocessing/semaphore.c | 15 ++++++--- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index 7c855113113c20..64e666b5af6f5b 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() #if defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) @@ -75,7 +76,9 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ } timeout_obj = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock_acquire_impl(self, blocking, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -100,7 +103,13 @@ _multiprocessing_SemLock_release_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock_release(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock_release_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_release_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) */ @@ -172,7 +181,9 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ } timeout_obj = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock_acquire_impl(self, blocking, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -197,7 +208,13 @@ _multiprocessing_SemLock_release_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock_release(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock_release_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_release_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) && !defined(MS_WINDOWS) */ @@ -340,7 +357,13 @@ _multiprocessing_SemLock__count_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock__count(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__count_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock__count_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -542,4 +565,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=d57992037e6770b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=713b597256233716 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index f8f2afda28d06d..5bb055f501e35b 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -81,6 +81,7 @@ _GetSemaphoreValue(HANDLE handle, long *value) } /*[clinic input] +@critical_section _multiprocessing.SemLock.acquire block as blocking: bool = True @@ -92,7 +93,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=079ca779975f3ad6]*/ { double timeout; DWORD res, full_msecs, nhandles; @@ -172,6 +173,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, } /*[clinic input] +@critical_section _multiprocessing.SemLock.release Release the semaphore/lock. @@ -179,7 +181,7 @@ Release the semaphore/lock. static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self) -/*[clinic end generated code: output=b22f53ba96b0d1db input=ba7e63a961885d3d]*/ +/*[clinic end generated code: output=b22f53ba96b0d1db input=9bd62d3645e7a531]*/ { if (self->kind == RECURSIVE_MUTEX) { if (!ISMINE(self)) { @@ -297,6 +299,7 @@ sem_timedwait_save(sem_t *sem, struct timespec *deadline, PyThreadState *_save) #endif /* !HAVE_SEM_TIMEDWAIT */ /*[clinic input] +@critical_section _multiprocessing.SemLock.acquire block as blocking: bool = True @@ -308,7 +311,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=079ca779975f3ad6]*/ { int res, err = 0; struct timespec deadline = {0}; @@ -382,6 +385,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, } /*[clinic input] +@critical_section _multiprocessing.SemLock.release Release the semaphore/lock. @@ -389,7 +393,7 @@ Release the semaphore/lock. static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self) -/*[clinic end generated code: output=b22f53ba96b0d1db input=ba7e63a961885d3d]*/ +/*[clinic end generated code: output=b22f53ba96b0d1db input=9bd62d3645e7a531]*/ { if (self->kind == RECURSIVE_MUTEX) { if (!ISMINE(self)) { @@ -583,6 +587,7 @@ semlock_dealloc(SemLockObject* self) } /*[clinic input] +@critical_section _multiprocessing.SemLock._count Num of `acquire()`s minus num of `release()`s for this process. @@ -590,7 +595,7 @@ Num of `acquire()`s minus num of `release()`s for this process. static PyObject * _multiprocessing_SemLock__count_impl(SemLockObject *self) -/*[clinic end generated code: output=5ba8213900e517bb input=36fc59b1cd1025ab]*/ +/*[clinic end generated code: output=5ba8213900e517bb input=9fa6e0b321b16935]*/ { return PyLong_FromLong((long)self->count); } From 42205143f8b3211d1392f1d9f2cf6717bdaa5b47 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 4 Apr 2024 14:10:46 -0400 Subject: [PATCH 085/143] gh-117478: Add `@support.requires_gil_enabled` decorator (#117479) Co-authored-by: Kirill Podoprigora --- Doc/library/test.rst | 6 ++++++ Lib/test/support/__init__.py | 7 ++++++- Lib/test/test_capi/test_mem.py | 4 ++-- Lib/test/test_cext/__init__.py | 4 ++-- Lib/test/test_concurrent_futures/test_process_pool.py | 2 +- Lib/test/test_concurrent_futures/test_thread_pool.py | 2 +- Lib/test/test_gc.py | 9 +++++---- 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 7d28f625345726..92d675b48690ff 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -731,6 +731,12 @@ The :mod:`test.support` module defines the following functions: macOS version is less than the minimum, the test is skipped. +.. decorator:: requires_gil_enabled + + Decorator for skipping tests on the free-threaded build. If the + :term:`GIL` is disabled, the test is skipped. + + .. decorator:: requires_IEEE_754 Decorator for skipping tests on non-IEEE 754 platforms. diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d7bc416ab04086..2be9cd099a68d6 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -29,7 +29,7 @@ "captured_stdin", "captured_stderr", # unittest "is_resource_enabled", "requires", "requires_freebsd_version", - "requires_linux_version", "requires_mac_ver", + "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", "requires_gzip", "requires_bz2", "requires_lzma", "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", @@ -837,6 +837,11 @@ def check_cflags_pgo(): Py_GIL_DISABLED = bool(sysconfig.get_config_var('Py_GIL_DISABLED')) + +def requires_gil_enabled(msg="needs the GIL enabled"): + """Decorator for skipping tests on the free-threaded build.""" + return unittest.skipIf(Py_GIL_DISABLED, msg) + if Py_GIL_DISABLED: _header = 'PHBBInP' else: diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 1958ecc0df4a7c..296601e8ee4f5f 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -152,8 +152,8 @@ class C(): pass self.assertGreaterEqual(count, i*10-4) -# Py_GIL_DISABLED requires mimalloc (not malloc) -@unittest.skipIf(support.Py_GIL_DISABLED, 'need malloc') +# free-threading requires mimalloc (not malloc) +@support.requires_gil_enabled class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index e4472b3355759c..ec44b0ce1f8a56 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -40,11 +40,11 @@ def test_build_c11(self): def test_build_c99(self): self.check_build('_test_c99_cext', std='c99') - @unittest.skipIf(support.Py_GIL_DISABLED, 'incompatible with Free Threading') + @support.requires_gil_enabled('incompatible with Free Threading') def test_build_limited(self): self.check_build('_test_limited_cext', limited=True) - @unittest.skipIf(support.Py_GIL_DISABLED, 'broken for now with Free Threading') + @support.requires_gil_enabled('broken for now with Free Threading') def test_build_limited_c11(self): self.check_build('_test_limited_c11_cext', limited=True, std='c11') diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index e60e7a6607a997..8b1bdaa33d8f5c 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -116,7 +116,7 @@ def test_saturation(self): for _ in range(job_count): sem.release() - @unittest.skipIf(support.Py_GIL_DISABLED, "gh-117344: test is flaky without the GIL") + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_process_reuse_one(self): executor = self.executor assert executor._max_workers >= 4 diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 86e65265516c3f..2b5bea9f4055a2 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -41,7 +41,7 @@ def acquire_lock(lock): sem.release() executor.shutdown(wait=True) - @unittest.skipIf(support.Py_GIL_DISABLED, "gh-117344: test is flaky without the GIL") + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_thread_reuse(self): executor = self.executor_type() executor.submit(mul, 21, 2).result() diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 3a01013b771082..8a748cb55538e8 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,7 +1,8 @@ import unittest import unittest.mock from test.support import (verbose, refcount_test, - cpython_only, requires_subprocess, Py_GIL_DISABLED) + cpython_only, requires_subprocess, + requires_gil_enabled) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script @@ -362,7 +363,7 @@ def __del__(self): # To minimize variations, though, we first store the get_count() results # and check them at the end. @refcount_test - @unittest.skipIf(Py_GIL_DISABLED, 'needs precise allocation counts') + @requires_gil_enabled('needs precise allocation counts') def test_get_count(self): gc.collect() a, b, c = gc.get_count() @@ -815,7 +816,7 @@ def test_get_objects(self): any(l is element for element in gc.get_objects()) ) - @unittest.skipIf(Py_GIL_DISABLED, 'need generational GC') + @requires_gil_enabled('need generational GC') def test_get_objects_generations(self): gc.collect() l = [] @@ -1046,7 +1047,7 @@ def setUp(self): def tearDown(self): gc.disable() - @unittest.skipIf(Py_GIL_DISABLED, "Free threading does not support incremental GC") + @requires_gil_enabled("Free threading does not support incremental GC") # Use small increments to emulate longer running process in a shorter time @gc_threshold(200, 10) def test_incremental_gc_handles_fast_cycle_creation(self): From 434bc593df4c0274b8afd3c1dcdc9234f469d9bf Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Thu, 4 Apr 2024 12:26:07 -0700 Subject: [PATCH 086/143] gh-112075: Make _PyDict_LoadGlobal thread safe (#117529) Make _PyDict_LoadGlobal threadsafe --- Include/internal/pycore_dict.h | 1 + Objects/dictobject.c | 42 ++++++++++++++-------------------- Objects/odictobject.c | 6 ++++- Python/bytecodes.c | 1 - Python/executor_cases.c.h | 1 - Python/generated_cases.c.h | 1 - 6 files changed, 23 insertions(+), 29 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 5507bdd539cc42..fba0dfc40714ec 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -97,6 +97,7 @@ extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); * -1 when no entry found, -3 when compare raises error. */ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 58a3d979339c49..b62d39ad6c5192 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1065,7 +1065,6 @@ compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, assert(ep->me_key != NULL); assert(PyUnicode_CheckExact(ep->me_key)); assert(!PyUnicode_CheckExact(key)); - // TODO: Thread safety if (unicode_get_hash(ep->me_key) == hash) { PyObject *startkey = ep->me_key; @@ -1192,7 +1191,8 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu PyDictKeysObject *dk; DictKeysKind kind; Py_ssize_t ix; - // TODO: Thread safety + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); start: dk = mp->ma_keys; kind = dk->dk_kind; @@ -1390,7 +1390,7 @@ dictkeys_generic_lookup_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObj return do_lookup(mp, dk, key, hash, compare_generic_threadsafe); } -static Py_ssize_t +Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) { PyDictKeysObject *dk; @@ -1488,6 +1488,16 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb return ix; } +#else // Py_GIL_DISABLED + +Py_ssize_t +_Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) +{ + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, value_addr); + Py_XNewRef(*value_addr); + return ix; +} + #endif int @@ -2343,11 +2353,12 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) * Raise an exception and return NULL if an error occurred (ex: computing the * key hash failed, key comparison failed, ...). Return NULL if the key doesn't * exist. Return the value if the key exists. + * + * Returns a new reference. */ PyObject * _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) { - // TODO: Thread safety Py_ssize_t ix; Py_hash_t hash; PyObject *value; @@ -2359,14 +2370,14 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) } /* namespace 1: globals */ - ix = _Py_dict_lookup(globals, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(globals, key, hash, &value); if (ix == DKIX_ERROR) return NULL; if (ix != DKIX_EMPTY && value != NULL) return value; /* namespace 2: builtins */ - ix = _Py_dict_lookup(builtins, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(builtins, key, hash, &value); assert(ix >= 0 || value == NULL); return value; } @@ -3214,11 +3225,7 @@ dict_subscript(PyObject *self, PyObject *key) if (hash == -1) return NULL; } -#ifdef Py_GIL_DISABLED ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); -#else - ix = _Py_dict_lookup(mp, key, hash, &value); -#endif if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) { @@ -3238,11 +3245,7 @@ dict_subscript(PyObject *self, PyObject *key) _PyErr_SetKeyError(key); return NULL; } -#ifdef Py_GIL_DISABLED return value; -#else - return Py_NewRef(value); -#endif } static int @@ -4109,24 +4112,13 @@ dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) if (hash == -1) return NULL; } -#ifdef Py_GIL_DISABLED ix = _Py_dict_lookup_threadsafe(self, key, hash, &val); -#else - ix = _Py_dict_lookup(self, key, hash, &val); -#endif if (ix == DKIX_ERROR) return NULL; -#ifdef Py_GIL_DISABLED if (ix == DKIX_EMPTY || val == NULL) { val = Py_NewRef(default_value); } return val; -#else - if (ix == DKIX_EMPTY || val == NULL) { - val = default_value; - } - return Py_NewRef(val); -#endif } static int diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 421bc52992d735..53f64fc81e7deb 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -535,8 +535,12 @@ _odict_get_index_raw(PyODictObject *od, PyObject *key, Py_hash_t hash) PyObject *value = NULL; PyDictKeysObject *keys = ((PyDictObject *)od)->ma_keys; Py_ssize_t ix; - +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe((PyDictObject *)od, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup((PyDictObject *)od, key, hash, &value); +#endif if (ix == DKIX_EMPTY) { return keys->dk_nentries; /* index of new entry */ } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8af48d9a0129b6..d6fb66a7be34ac 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1427,7 +1427,6 @@ dummy_func( } ERROR_IF(true, error); } - Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 8c3d41b64b49a5..a6e28f69881c1c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1254,7 +1254,6 @@ } if (true) JUMP_TO_ERROR(); } - Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0116acd5ae302f..a7764b0ec12e10 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4256,7 +4256,6 @@ } if (true) goto error; } - Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ From 63998a1347f3970ea4c69c881db69fc72b16a54c Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 4 Apr 2024 16:27:14 -0400 Subject: [PATCH 087/143] gh-117474: Skip GIL test in free-threaded build (#117475) In the free-threaded build, the GIL will typically be disabled so `py-bt` will not show threads waiting on the GIL. --- Lib/test/test_gdb/test_backtrace.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_gdb/test_backtrace.py b/Lib/test/test_gdb/test_backtrace.py index c41e7cb7c210de..fe67bf9ecc8880 100644 --- a/Lib/test/test_gdb/test_backtrace.py +++ b/Lib/test/test_gdb/test_backtrace.py @@ -49,6 +49,7 @@ def test_bt_full(self): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") + @support.requires_gil_enabled @support.requires_resource('cpu') def test_threads(self): 'Verify that "py-bt" indicates threads that are waiting for the GIL' From b5e60918afa53dfd59ad26a9f4b5207a9b304bc1 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 4 Apr 2024 17:14:35 -0400 Subject: [PATCH 088/143] gh-117549: Match declaration order for _Py_BackoffCounter initializer (#117551) Otherwise it might not compile with C++ (or certain C compilers/flags?). --- Include/internal/pycore_backoff.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 5d93c889e84976..decf92bc419c04 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -44,7 +44,7 @@ make_backoff_counter(uint16_t value, uint16_t backoff) { assert(backoff <= 15); assert(value <= 0xFFF); - return (_Py_BackoffCounter){.value = value, .backoff = backoff}; + return (_Py_BackoffCounter){.backoff = backoff, .value = value}; } static inline _Py_BackoffCounter From 0edde64a41c2c3eeb4fd495efe7fff3d631cae4b Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 4 Apr 2024 18:49:18 -0400 Subject: [PATCH 089/143] GH-117457: Correct pystats uop "miss" counts (GH-117477) --- Python/ceval.c | 2 - Python/executor_cases.c.h | 670 ++++++++++++++++++----- Tools/cases_generator/tier2_generator.py | 10 +- Tools/jit/template.c | 10 +- 4 files changed, 548 insertions(+), 144 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 57ae08ee3cf85a..f718a77fb029cb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1083,7 +1083,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } #endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - UOP_STAT_INC(uopcode, miss); Py_DECREF(current_executor); tstate->previous_executor = NULL; DISPATCH(); @@ -1091,7 +1090,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int exit_to_trace: assert(next_uop[-1].format == UOP_FORMAT_EXIT); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - UOP_STAT_INC(uopcode, miss); uint32_t exit_index = next_uop[-1].exit_index; assert(exit_index < current_executor->exit_count); _PyExitData *exit = ¤t_executor->exits[exit_index]; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a6e28f69881c1c..9c6e42a1a8e54f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14,13 +14,19 @@ case _RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) JUMP_TO_JUMP_TARGET(); + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) JUMP_TO_JUMP_TARGET(); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -323,7 +329,10 @@ case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) JUMP_TO_JUMP_TARGET(); + if (!PyBool_Check(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); break; } @@ -332,7 +341,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -350,7 +362,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -363,7 +378,10 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) JUMP_TO_JUMP_TARGET(); + if (!Py_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -374,7 +392,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) JUMP_TO_JUMP_TARGET(); + if (!PyUnicode_CheckExact(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -415,8 +436,14 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) JUMP_TO_JUMP_TARGET(); - if (!PyLong_CheckExact(right)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyLong_CheckExact(right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -473,8 +500,14 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) JUMP_TO_JUMP_TARGET(); - if (!PyFloat_CheckExact(right)) JUMP_TO_JUMP_TARGET(); + if (!PyFloat_CheckExact(left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyFloat_CheckExact(right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -531,8 +564,14 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) JUMP_TO_JUMP_TARGET(); - if (!PyUnicode_CheckExact(right)) JUMP_TO_JUMP_TARGET(); + if (!PyUnicode_CheckExact(left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_CheckExact(right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -623,12 +662,24 @@ PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyList_CheckExact(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -646,14 +697,29 @@ PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyUnicode_CheckExact(str)) JUMP_TO_JUMP_TARGET(); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_CheckExact(str)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) JUMP_TO_JUMP_TARGET(); + if (PyUnicode_GET_LENGTH(str) <= index) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) JUMP_TO_JUMP_TARGET(); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -669,12 +735,24 @@ PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyTuple_CheckExact(tuple)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyTuple_CheckExact(tuple)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) JUMP_TO_JUMP_TARGET(); + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -692,7 +770,10 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); if (rc == 0) { @@ -757,13 +838,25 @@ sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyList_CheckExact(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); @@ -782,7 +875,10 @@ sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); @@ -1072,8 +1168,14 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; assert(oparg == 2); - if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); - if (PyTuple_GET_SIZE(seq) != 2) JUMP_TO_JUMP_TARGET(); + if (!PyTuple_CheckExact(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyTuple_GET_SIZE(seq) != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -1090,8 +1192,14 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); - if (PyTuple_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET(); + if (!PyTuple_CheckExact(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyTuple_GET_SIZE(seq) != oparg) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1108,8 +1216,14 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyList_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); - if (PyList_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyList_GET_SIZE(seq) != oparg) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1280,8 +1394,14 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); - if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (dict->ma_keys->dk_version != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1289,8 +1409,14 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); - if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (dict->ma_keys->dk_version != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1303,7 +1429,10 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - if (res == NULL) JUMP_TO_JUMP_TARGET(); + if (res == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1321,7 +1450,10 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - if (res == NULL) JUMP_TO_JUMP_TARGET(); + if (res == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1645,8 +1777,14 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); - if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1670,8 +1808,14 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); - if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1744,7 +1888,10 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - if (tp->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET(); + if (tp->tp_version_tag != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -1753,7 +1900,10 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET(); + if (!_PyObject_InlineValues(owner)->valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -1765,7 +1915,10 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); attr = _PyObject_InlineValues(owner)->values[index]; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1782,7 +1935,10 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); attr = _PyObject_InlineValues(owner)->values[index]; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1799,10 +1955,16 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); - if (!PyModule_CheckExact(owner)) JUMP_TO_JUMP_TARGET(); + if (!PyModule_CheckExact(owner)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != dict_version) JUMP_TO_JUMP_TARGET(); + if (dict->ma_keys->dk_version != dict_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -1818,7 +1980,10 @@ assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1835,7 +2000,10 @@ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); PyDictObject *dict = managed_dict->dict; - if (dict == NULL) JUMP_TO_JUMP_TARGET(); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(PyDict_CheckExact((PyObject *)dict)); break; } @@ -1849,19 +2017,31 @@ uint16_t hint = (uint16_t)CURRENT_OPERAND(); PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); PyDictObject *dict = managed_dict->dict; - if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET(); + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) JUMP_TO_JUMP_TARGET(); + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) JUMP_TO_JUMP_TARGET(); + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } attr = ep->me_value; } - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1881,7 +2061,10 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1899,7 +2082,10 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1916,9 +2102,15 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - if (!PyType_Check(owner)) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(owner)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(type_version != 0); - if (((PyTypeObject *)owner)->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET(); + if (((PyTypeObject *)owner)->tp_version_tag != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -1967,8 +2159,14 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_ManagedDictPointer(owner)->dict) JUMP_TO_JUMP_TARGET(); - if (_PyObject_InlineValues(owner)->valid == 0) JUMP_TO_JUMP_TARGET(); + if (_PyObject_ManagedDictPointer(owner)->dict) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (_PyObject_InlineValues(owner)->valid == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2063,8 +2261,14 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!_PyLong_IsCompact((PyLongObject *)left)) JUMP_TO_JUMP_TARGET(); - if (!_PyLong_IsCompact((PyLongObject *)right)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsCompact((PyLongObject *)left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_IsCompact((PyLongObject *)right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -2143,7 +2347,10 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) JUMP_TO_JUMP_TARGET(); + if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! int res = _PySet_Contains((PySetObject *)right, left); @@ -2163,7 +2370,10 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyDict_CheckExact(right)) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CONTAINS_OP, hit); int res = PyDict_Contains(right, left); Py_DECREF(left); @@ -2383,7 +2593,10 @@ Py_DECREF(iter); STACK_SHRINK(1); /* The translator sets the deopt target just past END_FOR */ - if (true) JUMP_TO_JUMP_TARGET(); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } // Common case: no jump, leave it to the code generator stack_pointer[0] = next; @@ -2396,7 +2609,10 @@ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyListIter_Type) JUMP_TO_JUMP_TARGET(); + if (Py_TYPE(iter) != &PyListIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2408,8 +2624,14 @@ _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL) JUMP_TO_JUMP_TARGET(); - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET(); + if (seq == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2431,7 +2653,10 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyTupleIter_Type) JUMP_TO_JUMP_TARGET(); + if (Py_TYPE(iter) != &PyTupleIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2443,8 +2668,14 @@ _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL) JUMP_TO_JUMP_TARGET(); - if (it->it_index >= PyTuple_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET(); + if (seq == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (it->it_index >= PyTuple_GET_SIZE(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2467,7 +2698,10 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - if (Py_TYPE(r) != &PyRangeIter_Type) JUMP_TO_JUMP_TARGET(); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2478,7 +2712,10 @@ iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) JUMP_TO_JUMP_TARGET(); + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2565,7 +2802,10 @@ PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET(); + if (!_PyObject_InlineValues(owner)->valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2575,7 +2815,10 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) JUMP_TO_JUMP_TARGET(); + if (owner_heap_type->ht_cached_keys->dk_version != keys_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2657,7 +2900,10 @@ char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ - if (dict != NULL) JUMP_TO_JUMP_TARGET(); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2695,8 +2941,14 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (null != NULL) JUMP_TO_JUMP_TARGET(); - if (Py_TYPE(callable) != &PyMethod_Type) JUMP_TO_JUMP_TARGET(); + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2718,7 +2970,10 @@ } case _CHECK_PEP_523: { - if (tstate->interp->eval_frame) JUMP_TO_JUMP_TARGET(); + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2729,11 +2984,20 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (!PyFunction_Check(callable)) JUMP_TO_JUMP_TARGET(); + if (!PyFunction_Check(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyFunctionObject *func = (PyFunctionObject *)callable; - if (func->func_version != func_version) JUMP_TO_JUMP_TARGET(); + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (self_or_null != NULL)) JUMP_TO_JUMP_TARGET(); + if (code->co_argcount != oparg + (self_or_null != NULL)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2743,8 +3007,14 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) JUMP_TO_JUMP_TARGET(); - if (tstate->py_recursion_remaining <= 1) JUMP_TO_JUMP_TARGET(); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2926,8 +3196,14 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) JUMP_TO_JUMP_TARGET(); - if (callable != (PyObject *)&PyType_Type) JUMP_TO_JUMP_TARGET(); + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable != (PyObject *)&PyType_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(arg)); Py_DECREF(arg); @@ -2946,8 +3222,14 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) JUMP_TO_JUMP_TARGET(); - if (callable != (PyObject *)&PyUnicode_Type) JUMP_TO_JUMP_TARGET(); + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable != (PyObject *)&PyUnicode_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); res = PyObject_Str(arg); Py_DECREF(arg); @@ -2967,8 +3249,14 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) JUMP_TO_JUMP_TARGET(); - if (callable != (PyObject *)&PyTuple_Type) JUMP_TO_JUMP_TARGET(); + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable != (PyObject *)&PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); res = PySequence_Tuple(arg); Py_DECREF(arg); @@ -3008,9 +3296,15 @@ args--; total_args++; } - if (!PyType_Check(callable)) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyTypeObject *tp = (PyTypeObject *)callable; - if (tp->tp_vectorcall == NULL) JUMP_TO_JUMP_TARGET(); + if (tp->tp_vectorcall == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -3039,11 +3333,23 @@ args--; total_args++; } - if (total_args != 1) JUMP_TO_JUMP_TARGET(); - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); - if (PyCFunction_GET_FLAGS(callable) != METH_O) JUMP_TO_JUMP_TARGET(); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyCFunction_CheckExact(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable) != METH_O) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; @@ -3074,8 +3380,14 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); - if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); + if (!PyCFunction_CheckExact(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -3110,8 +3422,14 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); - if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); + if (!PyCFunction_CheckExact(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = @@ -3145,9 +3463,15 @@ args--; total_args++; } - if (total_args != 1) JUMP_TO_JUMP_TARGET(); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.len) JUMP_TO_JUMP_TARGET(); + if (callable != interp->callable_cache.len) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); @@ -3181,9 +3505,15 @@ args--; total_args++; } - if (total_args != 2) JUMP_TO_JUMP_TARGET(); + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.isinstance) JUMP_TO_JUMP_TARGET(); + if (callable != interp->callable_cache.isinstance) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; @@ -3219,15 +3549,30 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (total_args != 2) JUMP_TO_JUMP_TARGET(); - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) JUMP_TO_JUMP_TARGET(); + if (meth->ml_flags != METH_O) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyObject *arg = args[1]; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3258,12 +3603,21 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, d_type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(self, d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = @@ -3296,15 +3650,30 @@ args--; total_args++; } - if (total_args != 1) JUMP_TO_JUMP_TARGET(); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); - if (meth->ml_flags != METH_NOARGS) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (meth->ml_flags != METH_NOARGS) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3335,11 +3704,20 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); + if (meth->ml_flags != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; @@ -3543,7 +3921,10 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsTrue(flag)) JUMP_TO_JUMP_TARGET(); + if (!Py_IsTrue(flag)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(Py_IsTrue(flag)); break; } @@ -3552,7 +3933,10 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsFalse(flag)) JUMP_TO_JUMP_TARGET(); + if (!Py_IsFalse(flag)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(Py_IsFalse(flag)); break; } @@ -3563,7 +3947,10 @@ stack_pointer += -1; if (!Py_IsNone(val)) { Py_DECREF(val); - if (1) JUMP_TO_JUMP_TARGET(); + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } @@ -3572,7 +3959,10 @@ PyObject *val; val = stack_pointer[-1]; stack_pointer += -1; - if (Py_IsNone(val)) JUMP_TO_JUMP_TARGET(); + if (Py_IsNone(val)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_DECREF(val); break; } @@ -3611,12 +4001,18 @@ } case _EXIT_TRACE: { - if (1) JUMP_TO_JUMP_TARGET(); + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _CHECK_VALIDITY: { - if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET(); + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -3676,7 +4072,10 @@ case _CHECK_FUNCTION: { uint32_t func_version = (uint32_t)CURRENT_OPERAND(); assert(PyFunction_Check(frame->f_funcobj)); - if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) JUMP_TO_JUMP_TARGET(); + if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -3743,7 +4142,10 @@ case _CHECK_VALIDITY_AND_SET_IP: { PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); - if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET(); + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 114d28ee745632..944d134f12a18e 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -100,7 +100,10 @@ def tier2_replace_deopt( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") JUMP_TO_JUMP_TARGET();\n") + out.emit(") {\n") + out.emit("UOP_STAT_INC(uopcode, miss);\n") + out.emit("JUMP_TO_JUMP_TARGET();\n"); + out.emit("}\n") def tier2_replace_exit_if( @@ -115,7 +118,10 @@ def tier2_replace_exit_if( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") JUMP_TO_JUMP_TARGET();\n") + out.emit(") {\n") + out.emit("UOP_STAT_INC(uopcode, miss);\n") + out.emit("JUMP_TO_JUMP_TARGET();\n") + out.emit("}\n") def tier2_replace_oparg( diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 351bc2f3dd48de..2300bd0f1f31ec 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -85,7 +85,7 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * // Locals that the instruction implementations expect to exist: PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) int oparg; - int opcode = _JIT_OPCODE; + int uopcode = _JIT_OPCODE; // Other stuff we need handy: PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) @@ -93,14 +93,14 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX) OPT_STAT_INC(uops_executed); - UOP_STAT_INC(opcode, execution_count); + UOP_STAT_INC(uopcode, execution_count); // The actual instruction definitions (only one will be used): - if (opcode == _JUMP_TO_TOP) { + if (uopcode == _JUMP_TO_TOP) { CHECK_EVAL_BREAKER(); PATCH_JUMP(_JIT_TOP); } - switch (opcode) { + switch (uopcode) { #include "executor_cases.c.h" default: Py_UNREACHABLE(); @@ -113,11 +113,9 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * GOTO_TIER_ONE(NULL); exit_to_tier1: tstate->previous_executor = (PyObject *)current_executor; - UOP_STAT_INC(opcode, miss); GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); exit_to_trace: { - UOP_STAT_INC(opcode, miss); _PyExitData *exit = ¤t_executor->exits[_exit_index]; Py_INCREF(exit->executor); tstate->previous_executor = (PyObject *)current_executor; From 9c1dfe21fdaf2d93c3e1d1bba1cbe240e35ff35d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 Apr 2024 10:29:15 +0200 Subject: [PATCH 090/143] gh-116303: Don't build xxlimited and xxlimited_35 if --disable-test-modules is given (#117554) --- configure | 4 ++-- configure.ac | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure b/configure index 542783e723d934..f9647566636e4c 100755 --- a/configure +++ b/configure @@ -31185,7 +31185,7 @@ printf %s "checking for stdlib extension module xxlimited... " >&6; } if test "$py_cv_module_xxlimited" != "n/a" then : - if true + if test "$TEST_MODULES" = yes then : if test "$ac_cv_func_dlopen" = yes then : @@ -31223,7 +31223,7 @@ printf %s "checking for stdlib extension module xxlimited_35... " >&6; } if test "$py_cv_module_xxlimited_35" != "n/a" then : - if true + if test "$TEST_MODULES" = yes then : if test "$ac_cv_func_dlopen" = yes then : diff --git a/configure.ac b/configure.ac index fc62bfe5a1d4c4..e195e15b39ed21 100644 --- a/configure.ac +++ b/configure.ac @@ -7661,8 +7661,8 @@ PY_STDLIB_MOD([_ctypes_test], dnl Limited API template modules. dnl Emscripten does not support shared libraries yet. -PY_STDLIB_MOD([xxlimited], [], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([xxlimited_35], [], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) From 757b62493b47c6d2f07fc8ecaa2278a7c8a3bea6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 Apr 2024 12:13:00 +0200 Subject: [PATCH 091/143] gh-117457: Regen executor cases post PR #117477 (#117559) --- Python/executor_cases.c.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9c6e42a1a8e54f..a3447da00477ca 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3984,8 +3984,14 @@ case _CHECK_STACK_SPACE_OPERAND: { uint32_t framesize = (uint32_t)CURRENT_OPERAND(); assert(framesize <= INT_MAX); - if (!_PyThreadState_HasStackSpace(tstate, framesize)) JUMP_TO_JUMP_TARGET(); - if (tstate->py_recursion_remaining <= 1) JUMP_TO_JUMP_TARGET(); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } From 9ceaee74db7da0e71042ab0b385d844e9f282adb Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 5 Apr 2024 13:55:59 +0200 Subject: [PATCH 092/143] gh-116608: importlib.resources: Un-deprecate functional API & add subdirectory support (GH-116609) --- Doc/library/importlib.resources.rst | 178 ++++++++++++++ Doc/whatsnew/3.13.rst | 39 ++-- Lib/importlib/resources/__init__.py | 17 ++ Lib/importlib/resources/_functional.py | 85 +++++++ .../resources/test_functional.py | 219 ++++++++++++++++++ ...-03-11-17-04-55.gh-issue-116608.30f58-.rst | 10 + 6 files changed, 533 insertions(+), 15 deletions(-) create mode 100644 Lib/importlib/resources/_functional.py create mode 100644 Lib/test/test_importlib/resources/test_functional.py create mode 100644 Misc/NEWS.d/next/Library/2024-03-11-17-04-55.gh-issue-116608.30f58-.rst diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index a5adf0b8546dbf..9a5e4c76e7bd8f 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -97,3 +97,181 @@ for example, a package and its resources can be imported from a zip file using .. versionchanged:: 3.12 Added support for *traversable* representing a directory. + + +.. _importlib_resources_functional: + +Functional API +^^^^^^^^^^^^^^ + +A set of simplified, backwards-compatible helpers is available. +These allow common operations in a single function call. + +For all the following functions: + +- *anchor* is an :class:`~importlib.resources.Anchor`, + as in :func:`~importlib.resources.files`. + Unlike in ``files``, it may not be omitted. + +- *path_names* are components of a resource's path name, relative to + the anchor. + For example, to get the text of resource named ``info.txt``, use:: + + importlib.resources.read_text(my_module, "info.txt") + + Like :meth:`Traversable.joinpath `, + The individual components should use forward slashes (``/``) + as path separators. + For example, the following are equivalent:: + + importlib.resources.read_binary(my_module, "pics/painting.png") + importlib.resources.read_binary(my_module, "pics", "painting.png") + + For backward compatibility reasons, functions that read text require + an explicit *encoding* argument if multiple *path_names* are given. + For example, to get the text of ``info/chapter1.txt``, use:: + + importlib.resources.read_text(my_module, "info", "chapter1.txt", + encoding='utf-8') + +.. function:: open_binary(anchor, *path_names) + + Open the named resource for binary reading. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function returns a :class:`~typing.BinaryIO` object, + that is, a binary stream open for reading. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).open('rb') + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: open_text(anchor, *path_names, encoding='utf-8', errors='strict') + + Open the named resource for text reading. + By default, the contents are read as strict UTF-8. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + *encoding* and *errors* have the same meaning as in built-in :func:`open`. + + For backward compatibility reasons, the *encoding* argument must be given + explicitly if there are multiple *path_names*. + This limitation is scheduled to be removed in Python 3.15. + + This function returns a :class:`~typing.TextIO` object, + that is, a text stream open for reading. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).open('r', encoding=encoding) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: read_binary(anchor, *path_names) + + Read and return the contents of the named resource as :class:`bytes`. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).read_bytes() + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: read_text(anchor, *path_names, encoding='utf-8', errors='strict') + + Read and return the contents of the named resource as :class:`str`. + By default, the contents are read as strict UTF-8. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + *encoding* and *errors* have the same meaning as in built-in :func:`open`. + + For backward compatibility reasons, the *encoding* argument must be given + explicitly if there are multiple *path_names*. + This limitation is scheduled to be removed in Python 3.15. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).read_text(encoding=encoding) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: path(anchor, *path_names) + + Provides the path to the *resource* as an actual file system path. This + function returns a context manager for use in a :keyword:`with` statement. + The context manager provides a :class:`pathlib.Path` object. + + Exiting the context manager cleans up any temporary files created, e.g. + when the resource needs to be extracted from a zip file. + + For example, the :meth:`~pathlib.Path.stat` method requires + an actual file system path; it can be used like this:: + + with importlib.resources.path(anchor, "resource.txt") as fspath: + result = fspath.stat() + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + as_file(files(anchor).joinpath(*path_names)) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: is_resource(anchor, *path_names) + + Return ``True`` if the named resource exists, otherwise ``False``. + This function does not consider directories to be resources. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).is_file() + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: contents(anchor, *path_names) + + Return an iterable over the named items within the package or path. + The iterable returns names of resources (e.g. files) and non-resources + (e.g. directories) as :class:`str`. + The iterable does not recurse into subdirectories. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + for resource in files(anchor).joinpath(*path_names).iterdir(): + yield resource.name + + .. deprecated:: 3.11 + Prefer ``iterdir()`` as above, which offers more control over the + results and richer functionality. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 7f6a86efc61bf7..99a9545dd4e586 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -409,6 +409,30 @@ and only logged in :ref:`Python Development Mode ` or on :ref:`Python built on debug mode `. (Contributed by Victor Stinner in :gh:`62948`.) +importlib +--------- + +Previously deprecated :mod:`importlib.resources` functions are un-deprecated: + + * :func:`~importlib.resources.is_resource()` + * :func:`~importlib.resources.open_binary()` + * :func:`~importlib.resources.open_text()` + * :func:`~importlib.resources.path()` + * :func:`~importlib.resources.read_binary()` + * :func:`~importlib.resources.read_text()` + +All now allow for a directory (or tree) of resources, using multiple positional +arguments. + +For text-reading functions, the *encoding* and *errors* must now be given as +keyword arguments. + +The :func:`~importlib.resources.contents()` remains deprecated in favor of +the full-featured :class:`~importlib.resources.abc.Traversable` API. +However, there is now no plan to remove it. + +(Contributed by Petr Viktorin in :gh:`106532`.) + ipaddress --------- @@ -1357,21 +1381,6 @@ configparser importlib --------- -* Remove :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy - `_ - for migration advice. - (Contributed by Jason R. Coombs in :gh:`106532`.) - * Remove deprecated :meth:`~object.__getitem__` access for :class:`!importlib.metadata.EntryPoint` objects. (Contributed by Jason R. Coombs in :gh:`113175`.) diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index ae83cd07c4d4fb..ec4441c9116118 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -7,6 +7,16 @@ Anchor, ) +from ._functional import ( + contents, + is_resource, + open_binary, + open_text, + path, + read_binary, + read_text, +) + from .abc import ResourceReader @@ -16,4 +26,11 @@ 'ResourceReader', 'as_file', 'files', + 'contents', + 'is_resource', + 'open_binary', + 'open_text', + 'path', + 'read_binary', + 'read_text', ] diff --git a/Lib/importlib/resources/_functional.py b/Lib/importlib/resources/_functional.py new file mode 100644 index 00000000000000..9e3ea1547d486a --- /dev/null +++ b/Lib/importlib/resources/_functional.py @@ -0,0 +1,85 @@ +"""Simplified function-based API for importlib.resources""" + +import warnings + +from ._common import files, as_file + + +_MISSING = object() + + +def open_binary(anchor, *path_names): + """Open for binary reading the *resource* within *package*.""" + return _get_resource(anchor, path_names).open('rb') + + +def open_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Open for text reading the *resource* within *package*.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.open('r', encoding=encoding, errors=errors) + + +def read_binary(anchor, *path_names): + """Read and return contents of *resource* within *package* as bytes.""" + return _get_resource(anchor, path_names).read_bytes() + + +def read_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Read and return contents of *resource* within *package* as str.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.read_text(encoding=encoding, errors=errors) + + +def path(anchor, *path_names): + """Return the path to the *resource* as an actual file system path.""" + return as_file(_get_resource(anchor, path_names)) + + +def is_resource(anchor, *path_names): + """Return ``True`` if there is a resource named *name* in the package, + + Otherwise returns ``False``. + """ + return _get_resource(anchor, path_names).is_file() + + +def contents(anchor, *path_names): + """Return an iterable over the named resources within the package. + + The iterable returns :class:`str` resources (e.g. files). + The iterable does not recurse into subdirectories. + """ + warnings.warn( + "importlib.resources.contents is deprecated. " + "Use files(anchor).iterdir() instead.", + DeprecationWarning, + stacklevel=1, + ) + return ( + resource.name + for resource + in _get_resource(anchor, path_names).iterdir() + ) + + +def _get_encoding_arg(path_names, encoding): + # For compatibility with versions where *encoding* was a positional + # argument, it needs to be given explicitly when there are multiple + # *path_names*. + # This limitation can be removed in Python 3.15. + if encoding is _MISSING: + if len(path_names) > 1: + raise TypeError( + "'encoding' argument required with multiple path names", + ) + else: + return 'utf-8' + return encoding + + +def _get_resource(anchor, path_names): + if anchor is None: + raise TypeError("anchor must be module or string, got None") + return files(anchor).joinpath(*path_names) diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py new file mode 100644 index 00000000000000..fd02fc7c0e7b15 --- /dev/null +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -0,0 +1,219 @@ +import unittest +import os + +from test.support.warnings_helper import ignore_warnings, check_warnings + +import importlib.resources as resources + +# Since the functional API forwards to Traversable, we only test +# filesystem resources here -- not zip files, namespace packages etc. +# We do test for two kinds of Anchor, though. + + +class StringAnchorMixin: + anchor01 = 'test.test_importlib.resources.data01' + anchor02 = 'test.test_importlib.resources.data02' + + +class ModuleAnchorMixin: + from . import data01 as anchor01 + from . import data02 as anchor02 + + +class FunctionalAPIBase: + def _gen_resourcetxt_path_parts(self): + """Yield various names of a text file in anchor02, each in a subTest + """ + for path_parts in ( + ('subdirectory', 'subsubdir', 'resource.txt'), + ('subdirectory/subsubdir/resource.txt',), + ('subdirectory/subsubdir', 'resource.txt'), + ): + with self.subTest(path_parts=path_parts): + yield path_parts + + def test_read_text(self): + self.assertEqual( + resources.read_text(self.anchor01, 'utf-8.file'), + 'Hello, UTF-8 world!\n', + ) + self.assertEqual( + resources.read_text( + self.anchor02, 'subdirectory', 'subsubdir', 'resource.txt', + encoding='utf-8', + ), + 'a resource', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_text( + self.anchor02, *path_parts, encoding='utf-8', + ), + 'a resource', + ) + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.read_text(self.anchor01) + with self.assertRaises(OSError): + resources.read_text(self.anchor01, 'no-such-file') + with self.assertRaises(UnicodeDecodeError): + resources.read_text(self.anchor01, 'utf-16.file') + self.assertEqual( + resources.read_text( + self.anchor01, 'binary.file', encoding='latin1', + ), + '\x00\x01\x02\x03', + ) + self.assertEqual( + resources.read_text( + self.anchor01, 'utf-16.file', + errors='backslashreplace', + ), + 'Hello, UTF-16 world!\n'.encode('utf-16').decode( + errors='backslashreplace', + ), + ) + + def test_read_binary(self): + self.assertEqual( + resources.read_binary(self.anchor01, 'utf-8.file'), + b'Hello, UTF-8 world!\n', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_binary(self.anchor02, *path_parts), + b'a resource', + ) + + def test_open_text(self): + with resources.open_text(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_text( + self.anchor02, *path_parts, + encoding='utf-8', + ) as f: + self.assertEqual(f.read(), 'a resource') + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.open_text(self.anchor01) + with self.assertRaises(OSError): + resources.open_text(self.anchor01, 'no-such-file') + with resources.open_text(self.anchor01, 'utf-16.file') as f: + with self.assertRaises(UnicodeDecodeError): + f.read() + with resources.open_text( + self.anchor01, 'binary.file', encoding='latin1', + ) as f: + self.assertEqual(f.read(), '\x00\x01\x02\x03') + with resources.open_text( + self.anchor01, 'utf-16.file', + errors='backslashreplace', + ) as f: + self.assertEqual( + f.read(), + 'Hello, UTF-16 world!\n'.encode('utf-16').decode( + errors='backslashreplace', + ), + ) + + def test_open_binary(self): + with resources.open_binary(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_binary( + self.anchor02, *path_parts, + ) as f: + self.assertEqual(f.read(), b'a resource') + + def test_path(self): + with resources.path(self.anchor01, 'utf-8.file') as path: + with open(str(path)) as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + with resources.path(self.anchor01) as path: + with open(os.path.join(path, 'utf-8.file')) as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + + def test_is_resource(self): + is_resource = resources.is_resource + self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) + self.assertFalse(is_resource(self.anchor01, 'no_such_file')) + self.assertFalse(is_resource(self.anchor01)) + self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertTrue(is_resource(self.anchor02, *path_parts)) + + def test_contents(self): + is_resource = resources.is_resource + with check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01) + self.assertGreaterEqual( + set(c), + {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, + ) + with ( + self.assertRaises(OSError), + check_warnings((".*contents.*", DeprecationWarning)), + ): + list(resources.contents(self.anchor01, 'utf-8.file')) + for path_parts in self._gen_resourcetxt_path_parts(): + with ( + self.assertRaises(OSError), + check_warnings((".*contents.*", DeprecationWarning)), + ): + list(resources.contents(self.anchor01, *path_parts)) + with check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01, 'subdirectory') + self.assertGreaterEqual( + set(c), + {'binary.file'}, + ) + + @ignore_warnings(category=DeprecationWarning) + def test_common_errors(self): + for func in ( + resources.read_text, + resources.read_binary, + resources.open_text, + resources.open_binary, + resources.path, + resources.is_resource, + resources.contents, + ): + with self.subTest(func=func): + # Rejecting None anchor + with self.assertRaises(TypeError): + func(None) + # Rejecting invalid anchor type + with self.assertRaises((TypeError, AttributeError)): + func(1234) + # Unknown module + with self.assertRaises(ModuleNotFoundError): + func('$missing module$') + + def test_text_errors(self): + for func in ( + resources.read_text, + resources.open_text, + ): + with self.subTest(func=func): + # Multiple path arguments need explicit encoding argument. + with self.assertRaises(TypeError): + func( + self.anchor02, 'subdirectory', + 'subsubdir', 'resource.txt', + ) + + +class FunctionalAPITest_StringAnchor( + unittest.TestCase, FunctionalAPIBase, StringAnchorMixin, +): + pass + + +class FunctionalAPITest_ModuleAnchor( + unittest.TestCase, FunctionalAPIBase, ModuleAnchorMixin, +): + pass diff --git a/Misc/NEWS.d/next/Library/2024-03-11-17-04-55.gh-issue-116608.30f58-.rst b/Misc/NEWS.d/next/Library/2024-03-11-17-04-55.gh-issue-116608.30f58-.rst new file mode 100644 index 00000000000000..d1536bc47c3ee0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-11-17-04-55.gh-issue-116608.30f58-.rst @@ -0,0 +1,10 @@ +The :mod:`importlib.resources` functions +:func:`~importlib.resources.is_resource()`, +:func:`~importlib.resources.open_binary()`, +:func:`~importlib.resources.open_text()`, +:func:`~importlib.resources.path()`, +:func:`~importlib.resources.read_binary()`, and +:func:`~importlib.resources.read_text()` are un-deprecated, and support +subdirectories via multiple positional arguments. +The :func:`~importlib.resources.contents()` function also allows subdirectories, +but remains deprecated. From abfa16b44bb9426312613893b6e193b02ee0304f Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 5 Apr 2024 13:35:01 +0100 Subject: [PATCH 093/143] GH-114847: Speed up `posixpath.realpath()` (#114848) Apply the following optimizations to `posixpath.realpath()`: - Remove use of recursion - Construct child paths directly rather than using `join()` - Use `os.getcwd[b]()` rather than `abspath()` - Use `startswith(sep)` rather than `isabs()` - Use slicing rather than `split()` Co-authored-by: Petr Viktorin --- Lib/posixpath.py | 88 ++++++++++++------- Lib/test/test_posixpath.py | 9 ++ ...-02-01-08-09-20.gh-issue-114847.-JrWrR.rst | 1 + 3 files changed, 64 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-01-08-09-20.gh-issue-114847.-JrWrR.rst diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 76ee721bfb5e33..0e8bb5ab10d916 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -403,55 +403,66 @@ def realpath(filename, *, strict=False): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) - path, ok = _joinrealpath(filename[:0], filename, strict, {}) - return abspath(path) - -# Join two paths, normalizing and eliminating any symbolic links -# encountered in the second path. -# Two leading slashes are replaced by a single slash. -def _joinrealpath(path, rest, strict, seen): - if isinstance(path, bytes): + if isinstance(filename, bytes): sep = b'/' curdir = b'.' pardir = b'..' + getcwd = os.getcwdb else: sep = '/' curdir = '.' pardir = '..' + getcwd = os.getcwd + + # The stack of unresolved path parts. When popped, a special value of None + # indicates that a symlink target has been resolved, and that the original + # symlink path can be retrieved by popping again. The [::-1] slice is a + # very fast way of spelling list(reversed(...)). + rest = filename.split(sep)[::-1] + + # The resolved path, which is absolute throughout this function. + # Note: getcwd() returns a normalized and symlink-free path. + path = sep if filename.startswith(sep) else getcwd() - if rest.startswith(sep): - rest = rest[1:] - path = sep + # Mapping from symlink paths to *fully resolved* symlink targets. If a + # symlink is encountered but not yet resolved, the value is None. This is + # used both to detect symlink loops and to speed up repeated traversals of + # the same links. + seen = {} + + # Whether we're calling lstat() and readlink() to resolve symlinks. If we + # encounter an OSError for a symlink loop in non-strict mode, this is + # switched off. + querying = True while rest: - name, _, rest = rest.partition(sep) + name = rest.pop() + if name is None: + # resolved symlink target + seen[rest.pop()] = path + continue if not name or name == curdir: # current dir continue if name == pardir: # parent dir - if path: - parent, name = split(path) - if name == pardir: - # ../.. - path = join(path, pardir) - else: - # foo/bar/.. -> foo - path = parent - else: - # .. - path = pardir + path = path[:path.rindex(sep)] or sep + continue + if path == sep: + newpath = path + name + else: + newpath = path + sep + name + if not querying: + path = newpath continue - newpath = join(path, name) try: st = os.lstat(newpath) + if not stat.S_ISLNK(st.st_mode): + path = newpath + continue except OSError: if strict: raise - is_link = False - else: - is_link = stat.S_ISLNK(st.st_mode) - if not is_link: path = newpath continue # Resolve the symbolic link @@ -467,14 +478,23 @@ def _joinrealpath(path, rest, strict, seen): os.stat(newpath) else: # Return already resolved part + rest of the path unchanged. - return join(newpath, rest), False + path = newpath + querying = False + continue seen[newpath] = None # not resolved symlink - path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen) - if not ok: - return join(path, rest), False - seen[newpath] = path # resolved symlink + target = os.readlink(newpath) + if target.startswith(sep): + # Symlink target is absolute; reset resolved path. + path = sep + # Push the symlink path onto the stack, and signal its specialness by + # also pushing None. When these entries are popped, we'll record the + # fully-resolved symlink target in the 'seen' mapping. + rest.append(newpath) + rest.append(None) + # Push the unresolved symlink target parts onto the stack. + rest.extend(target.split(sep)[::-1]) - return path, True + return path supports_unicode_filenames = (sys.platform == 'darwin') diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index cbb7c4c52d9697..807f985f7f4df7 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -456,6 +456,15 @@ def test_realpath_relative(self): finally: os_helper.unlink(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_missing_pardir(self): + try: + os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN) + self.assertEqual(realpath("nonexistent/../" + os_helper.TESTFN), ABSTFN + "1") + finally: + os_helper.unlink(os_helper.TESTFN) + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_symlink_loops(self): diff --git a/Misc/NEWS.d/next/Library/2024-02-01-08-09-20.gh-issue-114847.-JrWrR.rst b/Misc/NEWS.d/next/Library/2024-02-01-08-09-20.gh-issue-114847.-JrWrR.rst new file mode 100644 index 00000000000000..bf011fed3efdbc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-01-08-09-20.gh-issue-114847.-JrWrR.rst @@ -0,0 +1 @@ +Speed up :func:`os.path.realpath` on non-Windows platforms. From 687616877ba540a44f82ff764b5f13d36c0f3910 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 5 Apr 2024 15:21:16 +0100 Subject: [PATCH 094/143] gh-111140: PyLong_From/AsNativeBytes: Take *flags* rather than just *endianness* (GH-116053) --- Doc/c-api/long.rst | 142 ++++++++++++------ Include/cpython/longobject.h | 24 ++- Lib/test/test_capi/test_long.py | 118 ++++++++++++++- ...-02-28-15-50-01.gh-issue-111140.mpwcUg.rst | 3 + Modules/_testcapi/long.c | 14 +- Objects/longobject.c | 106 ++++++++++--- 6 files changed, 319 insertions(+), 88 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-02-28-15-50-01.gh-issue-111140.mpwcUg.rst diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 6a7eba7761de1a..1eb8f191c3ca32 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -113,24 +113,28 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. retrieved from the resulting value using :c:func:`PyLong_AsVoidPtr`. -.. c:function:: PyObject* PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, int endianness) +.. c:function:: PyObject* PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, int flags) Create a Python integer from the value contained in the first *n_bytes* of *buffer*, interpreted as a two's-complement signed number. - *endianness* may be passed ``-1`` for the native endian that CPython was - compiled with, or else ``0`` for big endian and ``1`` for little. + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is a sign bit. Passing + ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` will produce the same result as calling + :c:func:`PyLong_FromUnsignedNativeBytes`. Other flags are ignored. .. versionadded:: 3.13 -.. c:function:: PyObject* PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n_bytes, int endianness) +.. c:function:: PyObject* PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n_bytes, int flags) Create a Python integer from the value contained in the first *n_bytes* of *buffer*, interpreted as an unsigned number. - *endianness* may be passed ``-1`` for the native endian that CPython was - compiled with, or else ``0`` for big endian and ``1`` for little. + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is not a sign bit. Flags other than endian are ignored. .. versionadded:: 3.13 @@ -354,14 +358,41 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. -.. c:function:: Py_ssize_t PyLong_AsNativeBytes(PyObject *pylong, void* buffer, Py_ssize_t n_bytes, int endianness) +.. c:function:: Py_ssize_t PyLong_AsNativeBytes(PyObject *pylong, void* buffer, Py_ssize_t n_bytes, int flags) - Copy the Python integer value to a native *buffer* of size *n_bytes*:: + Copy the Python integer value *pylong* to a native *buffer* of size + *n_bytes*. The *flags* can be set to ``-1`` to behave similarly to a C cast, + or to values documented below to control the behavior. + + Returns ``-1`` with an exception raised on error. This may happen if + *pylong* cannot be interpreted as an integer, or if *pylong* was negative + and the ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` flag was set. + + Otherwise, returns the number of bytes required to store the value. + If this is equal to or less than *n_bytes*, the entire value was copied. + All *n_bytes* of the buffer are written: large buffers are padded with + zeroes. + + If the returned value is greater than than *n_bytes*, the value was + truncated: as many of the lowest bits of the value as could fit are written, + and the higher bits are ignored. This matches the typical behavior + of a C-style downcast. + + .. note:: + + Overflow is not considered an error. If the returned value + is larger than *n_bytes*, most significant bits were discarded. + + ``0`` will never be returned. + + Values are always copied as two's-complement. + + Usage example:: int32_t value; Py_ssize_t bytes = PyLong_AsNativeBits(pylong, &value, sizeof(value), -1); if (bytes < 0) { - // A Python exception was set with the reason. + // Failed. A Python exception was set with the reason. return NULL; } else if (bytes <= (Py_ssize_t)sizeof(value)) { @@ -372,19 +403,24 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. // lowest bits of pylong. } - The above example may look *similar* to - :c:func:`PyLong_As* ` - but instead fills in a specific caller defined type and never raises an - error about of the :class:`int` *pylong*'s value regardless of *n_bytes* - or the returned byte count. + Passing zero to *n_bytes* will return the size of a buffer that would + be large enough to hold the value. This may be larger than technically + necessary, but not unreasonably so. - To get at the entire potentially big Python value, this can be used to - reserve enough space and copy it:: + .. note:: + + Passing *n_bytes=0* to this function is not an accurate way to determine + the bit length of a value. + + If *n_bytes=0*, *buffer* may be ``NULL``. + + To get at the entire Python value of an unknown size, the function can be + called twice: first to determine the buffer size, then to fill it:: // Ask how much space we need. Py_ssize_t expected = PyLong_AsNativeBits(pylong, NULL, 0, -1); if (expected < 0) { - // A Python exception was set with the reason. + // Failed. A Python exception was set with the reason. return NULL; } assert(expected != 0); // Impossible per the API definition. @@ -395,11 +431,11 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. } // Safely get the entire value. Py_ssize_t bytes = PyLong_AsNativeBits(pylong, bignum, expected, -1); - if (bytes < 0) { // Exception set. + if (bytes < 0) { // Exception has been set. free(bignum); return NULL; } - else if (bytes > expected) { // Be safe, should not be possible. + else if (bytes > expected) { // This should not be possible. PyErr_SetString(PyExc_RuntimeError, "Unexpected bignum truncation after a size check."); free(bignum); @@ -409,35 +445,51 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. // ... use bignum ... free(bignum); - *endianness* may be passed ``-1`` for the native endian that CPython was - compiled with, or ``0`` for big endian and ``1`` for little. - - Returns ``-1`` with an exception raised if *pylong* cannot be interpreted as - an integer. Otherwise, return the size of the buffer required to store the - value. If this is equal to or less than *n_bytes*, the entire value was - copied. ``0`` will never be returned. - - Unless an exception is raised, all *n_bytes* of the buffer will always be - written. In the case of truncation, as many of the lowest bits of the value - as could fit are written. This allows the caller to ignore all non-negative - results if the intent is to match the typical behavior of a C-style - downcast. No exception is set on truncation. - - Values are always copied as two's-complement and sufficient buffer will be - requested to include a sign bit. For example, this may cause an value that - fits into 8 bytes when treated as unsigned to request 9 bytes, even though - all eight bytes were copied into the buffer. What has been omitted is the - zero sign bit -- redundant if the caller's intention is to treat the value - as unsigned. - - Passing zero to *n_bytes* will return the size of a buffer that would - be large enough to hold the value. This may be larger than technically - necessary, but not unreasonably so. + *flags* is either ``-1`` (``Py_ASNATIVEBYTES_DEFAULTS``) to select defaults + that behave most like a C cast, or a combintation of the other flags in + the table below. + Note that ``-1`` cannot be combined with other flags. + + Currently, ``-1`` corresponds to + ``Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_UNSIGNED_BUFFER``. + + ============================================= ====== + Flag Value + ============================================= ====== + .. c:macro:: Py_ASNATIVEBYTES_DEFAULTS ``-1`` + .. c:macro:: Py_ASNATIVEBYTES_BIG_ENDIAN ``0`` + .. c:macro:: Py_ASNATIVEBYTES_LITTLE_ENDIAN ``1`` + .. c:macro:: Py_ASNATIVEBYTES_NATIVE_ENDIAN ``3`` + .. c:macro:: Py_ASNATIVEBYTES_UNSIGNED_BUFFER ``4`` + .. c:macro:: Py_ASNATIVEBYTES_REJECT_NEGATIVE ``8`` + ============================================= ====== + + Specifying ``Py_ASNATIVEBYTES_NATIVE_ENDIAN`` will override any other endian + flags. Passing ``2`` is reserved. + + By default, sufficient buffer will be requested to include a sign bit. + For example, when converting 128 with *n_bytes=1*, the function will return + 2 (or more) in order to store a zero sign bit. + + If ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` is specified, a zero sign bit + will be omitted from size calculations. This allows, for example, 128 to fit + in a single-byte buffer. If the destination buffer is later treated as + signed, a positive input value may become negative. + Note that the flag does not affect handling of negative values: for those, + space for a sign bit is always requested. + + Specifying ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` causes an exception to be set + if *pylong* is negative. Without this flag, negative values will be copied + provided there is enough space for at least one sign bit, regardless of + whether ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` was specified. .. note:: - Passing *n_bytes=0* to this function is not an accurate way to determine - the bit length of a value. + With the default *flags* (``-1``, or *UNSIGNED_BUFFER* without + *REJECT_NEGATIVE*), multiple Python integers can map to a single value + without overflow. For example, both ``255`` and ``-1`` fit a single-byte + buffer and set all its bits. + This matches typical C cast behavior. .. versionadded:: 3.13 diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 07251db6bcc203..189229ee1035d8 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -4,11 +4,24 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); +#define Py_ASNATIVEBYTES_DEFAULTS -1 +#define Py_ASNATIVEBYTES_BIG_ENDIAN 0 +#define Py_ASNATIVEBYTES_LITTLE_ENDIAN 1 +#define Py_ASNATIVEBYTES_NATIVE_ENDIAN 3 +#define Py_ASNATIVEBYTES_UNSIGNED_BUFFER 4 +#define Py_ASNATIVEBYTES_REJECT_NEGATIVE 8 + /* PyLong_AsNativeBytes: Copy the integer value to a native variable. buffer points to the first byte of the variable. n_bytes is the number of bytes available in the buffer. Pass 0 to request the required size for the value. - endianness is -1 for native endian, 0 for big endian or 1 for little. + flags is a bitfield of the following flags: + * 1 - little endian + * 2 - native endian + * 4 - unsigned destination (e.g. don't reject copying 255 into one byte) + * 8 - raise an exception for negative inputs + If flags is -1 (all bits set), native endian is used and value truncation + behaves most like C (allows negative inputs and allow MSB set). Big endian mode will write the most significant byte into the address directly referenced by buffer; little endian will write the least significant byte into that address. @@ -24,19 +37,20 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); calculate the bit length of an integer object. */ PyAPI_FUNC(Py_ssize_t) PyLong_AsNativeBytes(PyObject* v, void* buffer, - Py_ssize_t n_bytes, int endianness); + Py_ssize_t n_bytes, int flags); /* PyLong_FromNativeBytes: Create an int value from a native integer n_bytes is the number of bytes to read from the buffer. Passing 0 will always produce the zero int. PyLong_FromUnsignedNativeBytes always produces a non-negative int. - endianness is -1 for native endian, 0 for big endian or 1 for little. + flags is the same as for PyLong_AsNativeBytes, but only supports selecting + the endianness or forcing an unsigned buffer. Returns the int object, or NULL with an exception set. */ PyAPI_FUNC(PyObject*) PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, - int endianness); + int flags); PyAPI_FUNC(PyObject*) PyLong_FromUnsignedNativeBytes(const void* buffer, - size_t n_bytes, int endianness); + size_t n_bytes, int flags); PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index d2140154d811b4..83f894e552f983 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -483,8 +483,12 @@ def test_long_asnativebytes(self): (-MAX_USIZE, SZ + 1), (2**255-1, 32), (-(2**255-1), 32), + (2**255, 33), + (-(2**255), 33), # if you ask, we'll say 33, but 32 would do (2**256-1, 33), (-(2**256-1), 33), + (2**256, 33), + (-(2**256), 33), ]: with self.subTest(f"sizeof-{v:X}"): buffer = bytearray(b"\x5a") @@ -523,15 +527,17 @@ def test_long_asnativebytes(self): (-1, b'\xff' * 10, min(11, SZ)), (-42, b'\xd6', 1), (-42, b'\xff' * 10 + b'\xd6', min(11, SZ)), - # Extracts 255 into a single byte, but requests sizeof(Py_ssize_t) - (255, b'\xff', SZ), + # Extracts 255 into a single byte, but requests 2 + # (this is currently a special case, and "should" request SZ) + (255, b'\xff', 2), (255, b'\x00\xff', 2), (256, b'\x01\x00', 2), + (0x80, b'\x00' * 7 + b'\x80', min(8, SZ)), # Extracts successfully (unsigned), but requests 9 bytes (2**63, b'\x80' + b'\x00' * 7, 9), - # "Extracts", but requests 9 bytes - (-2**63, b'\x80' + b'\x00' * 7, 9), (2**63, b'\x00\x80' + b'\x00' * 7, 9), + # Extracts into 8 bytes, but if you provide 9 we'll say 9 + (-2**63, b'\x80' + b'\x00' * 7, 8), (-2**63, b'\xff\x80' + b'\x00' * 7, 9), (2**255-1, b'\x7f' + b'\xff' * 31, 32), @@ -548,10 +554,15 @@ def test_long_asnativebytes(self): (-(2**256-1), b'\x00' * 31 + b'\x01', 33), (-(2**256-1), b'\xff' + b'\x00' * 31 + b'\x01', 33), (-(2**256-1), b'\xff\xff' + b'\x00' * 31 + b'\x01', 33), + # However, -2**255 precisely will extract into 32 bytes and return + # success. For bigger buffers, it will still succeed, but will + # return 33 + (-(2**255), b'\x80' + b'\x00' * 31, 32), + (-(2**255), b'\xff\x80' + b'\x00' * 31, 33), # The classic "Windows HRESULT as negative number" case # HRESULT hr; - # PyLong_CopyBits(<-2147467259>, &hr, sizeof(HRESULT)) + # PyLong_AsNativeBytes(<-2147467259>, &hr, sizeof(HRESULT), -1) # assert(hr == E_FAIL) (-2147467259, b'\x80\x00\x40\x05', 4), ]: @@ -569,14 +580,102 @@ def test_long_asnativebytes(self): f"PyLong_AsNativeBytes(v, buffer, {n}, )") self.assertEqual(expect_le, buffer[:n], "") + # Test cases that do not request size for a sign bit when we pass the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag + for v, expect_be, expect_n in [ + (255, b'\xff', 1), + # We pass a 2 byte buffer so it just uses the whole thing + (255, b'\x00\xff', 2), + + (2**63, b'\x80' + b'\x00' * 7, 8), + # We pass a 9 byte buffer so it uses the whole thing + (2**63, b'\x00\x80' + b'\x00' * 7, 9), + + (2**256-1, b'\xff' * 32, 32), + # We pass a 33 byte buffer so it uses the whole thing + (2**256-1, b'\x00' + b'\xff' * 32, 33), + ]: + with self.subTest(f"{v:X}-{len(expect_be)}bytes-unsigned"): + n = len(expect_be) + buffer = bytearray(b"\xa5"*n) + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 4), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 5), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + + # Ensure Py_ASNATIVEBYTES_REJECT_NEGATIVE raises on negative value + with self.assertRaises(ValueError): + asnativebytes(-1, buffer, 0, 8) + # Check a few error conditions. These are validated in code, but are # unspecified in docs, so if we make changes to the implementation, it's # fine to just update these tests rather than preserve the behaviour. - with self.assertRaises(SystemError): - asnativebytes(1, buffer, 0, 2) with self.assertRaises(TypeError): asnativebytes('not a number', buffer, 0, -1) + def test_long_asnativebytes_fuzz(self): + import math + from random import Random + from _testcapi import ( + pylong_asnativebytes as asnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + + rng = Random() + # Allocate bigger buffer than actual values are going to be + buffer = bytearray(260) + + for _ in range(1000): + n = rng.randrange(1, 256) + bytes_be = bytes([ + # Ensure the most significant byte is nonzero + rng.randrange(1, 256), + *[rng.randrange(256) for _ in range(n - 1)] + ]) + bytes_le = bytes_be[::-1] + v = int.from_bytes(bytes_le, 'little') + + expect_1 = expect_2 = (SZ, n) + if bytes_be[0] & 0x80: + # All values are positive, so if MSB is set, expect extra bit + # when we request the size or have a large enough buffer + expect_1 = (SZ, n + 1) + # When passing Py_ASNATIVEBYTES_UNSIGNED_BUFFER, we expect the + # return to be exactly the right size. + expect_2 = (n,) + + try: + actual = asnativebytes(v, buffer, 0, -1) + self.assertIn(actual, expect_1) + + actual = asnativebytes(v, buffer, len(buffer), 0) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_be, buffer[-n:]) + + actual = asnativebytes(v, buffer, len(buffer), 1) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_le, buffer[:n]) + + actual = asnativebytes(v, buffer, n, 4) + self.assertIn(actual, expect_2, bytes_be.hex()) + actual = asnativebytes(v, buffer, n, 5) + self.assertIn(actual, expect_2, bytes_be.hex()) + except AssertionError as ex: + value_hex = ''.join(reversed([ + f'{b:02X}{"" if i % 8 else "_"}' + for i, b in enumerate(bytes_le, start=1) + ])).strip('_') + if support.verbose: + print() + print(n, 'bytes') + print('hex =', value_hex) + print('int =', v) + raise + raise AssertionError(f"Value: 0x{value_hex}") from ex + def test_long_fromnativebytes(self): import math from _testcapi import ( @@ -617,6 +716,11 @@ def test_long_fromnativebytes(self): self.assertEqual(expect_u, fromnativebytes(v_be, n, -1, 0), f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + # Swap the unsigned request for tests and use the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag instead + self.assertEqual(expect_u, fromnativebytes(v_be, n, 4, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C API/2024-02-28-15-50-01.gh-issue-111140.mpwcUg.rst b/Misc/NEWS.d/next/C API/2024-02-28-15-50-01.gh-issue-111140.mpwcUg.rst new file mode 100644 index 00000000000000..113db93d186009 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-02-28-15-50-01.gh-issue-111140.mpwcUg.rst @@ -0,0 +1,3 @@ +Add additional flags to :c:func:`PyLong_AsNativeBytes` and +:c:func:`PyLong_FromNativeBytes` to allow the caller to determine how to handle +edge cases around values that fill the entire buffer. diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 28dca01bee09a0..769c3909ea3fb1 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -52,8 +52,8 @@ pylong_asnativebytes(PyObject *module, PyObject *args) { PyObject *v; Py_buffer buffer; - Py_ssize_t n, endianness; - if (!PyArg_ParseTuple(args, "Ow*nn", &v, &buffer, &n, &endianness)) { + Py_ssize_t n, flags; + if (!PyArg_ParseTuple(args, "Ow*nn", &v, &buffer, &n, &flags)) { return NULL; } if (buffer.readonly) { @@ -66,7 +66,7 @@ pylong_asnativebytes(PyObject *module, PyObject *args) PyBuffer_Release(&buffer); return NULL; } - Py_ssize_t res = PyLong_AsNativeBytes(v, buffer.buf, n, (int)endianness); + Py_ssize_t res = PyLong_AsNativeBytes(v, buffer.buf, n, (int)flags); PyBuffer_Release(&buffer); return res >= 0 ? PyLong_FromSsize_t(res) : NULL; } @@ -76,8 +76,8 @@ static PyObject * pylong_fromnativebytes(PyObject *module, PyObject *args) { Py_buffer buffer; - Py_ssize_t n, endianness, signed_; - if (!PyArg_ParseTuple(args, "y*nnn", &buffer, &n, &endianness, &signed_)) { + Py_ssize_t n, flags, signed_; + if (!PyArg_ParseTuple(args, "y*nnn", &buffer, &n, &flags, &signed_)) { return NULL; } if (buffer.len < n) { @@ -86,8 +86,8 @@ pylong_fromnativebytes(PyObject *module, PyObject *args) return NULL; } PyObject *res = signed_ - ? PyLong_FromNativeBytes(buffer.buf, n, (int)endianness) - : PyLong_FromUnsignedNativeBytes(buffer.buf, n, (int)endianness); + ? PyLong_FromNativeBytes(buffer.buf, n, (int)flags) + : PyLong_FromUnsignedNativeBytes(buffer.buf, n, (int)flags); PyBuffer_Release(&buffer); return res; } diff --git a/Objects/longobject.c b/Objects/longobject.c index cc2fe11f31c430..c4ab064d688d67 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1083,18 +1083,17 @@ _fits_in_n_bits(Py_ssize_t v, Py_ssize_t n) static inline int _resolve_endianness(int *endianness) { - if (*endianness < 0) { + if (*endianness == -1 || (*endianness & 2)) { *endianness = PY_LITTLE_ENDIAN; + } else { + *endianness &= 1; } - if (*endianness != 0 && *endianness != 1) { - PyErr_SetString(PyExc_SystemError, "invalid 'endianness' value"); - return -1; - } + assert(*endianness == 0 || *endianness == 1); return 0; } Py_ssize_t -PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) +PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) { PyLongObject *v; union { @@ -1109,7 +1108,7 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) return -1; } - int little_endian = endianness; + int little_endian = flags; if (_resolve_endianness(&little_endian) < 0) { return -1; } @@ -1125,6 +1124,15 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) do_decref = 1; } + if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) + && _PyLong_IsNegative(v)) { + PyErr_SetString(PyExc_ValueError, "Cannot convert negative int"); + if (do_decref) { + Py_DECREF(v); + } + return -1; + } + if (_PyLong_IsCompact(v)) { res = 0; cv.v = _PyLong_CompactValue(v); @@ -1159,6 +1167,15 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) /* If we fit, return the requested number of bytes */ if (_fits_in_n_bits(cv.v, n * 8)) { res = n; + } else if (cv.v > 0 && _fits_in_n_bits(cv.v, n * 8 + 1)) { + /* Positive values with the MSB set do not require an + * additional bit when the caller's intent is to treat them + * as unsigned. */ + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + res = n; + } else { + res = n + 1; + } } } else { @@ -1199,17 +1216,55 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) _PyLong_AsByteArray(v, buffer, (size_t)n, little_endian, 1, 0); } - // More efficient calculation for number of bytes required? + /* Calculates the number of bits required for the *absolute* value + * of v. This does not take sign into account, only magnitude. */ size_t nb = _PyLong_NumBits((PyObject *)v); - /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up - * multiples of 8 to the next byte, but we add an implied bit for - * the sign and it cancels out. */ - size_t n_needed = (nb / 8) + 1; - res = (Py_ssize_t)n_needed; - if ((size_t)res != n_needed) { - PyErr_SetString(PyExc_OverflowError, - "value too large to convert"); + if (nb == (size_t)-1) { res = -1; + } else { + /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up + * multiples of 8 to the next byte, but we add an implied bit for + * the sign and it cancels out. */ + res = (Py_ssize_t)(nb / 8) + 1; + } + + /* Two edge cases exist that are best handled after extracting the + * bits. These may result in us reporting overflow when the value + * actually fits. + */ + if (n > 0 && res == n + 1 && nb % 8 == 0) { + if (_PyLong_IsNegative(v)) { + /* Values of 0x80...00 from negative values that use every + * available bit in the buffer do not require an additional + * bit to store the sign. */ + int is_edge_case = 1; + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = 0; i < n && is_edge_case; ++i, ++b) { + if (i == 0) { + is_edge_case = (*b == (little_endian ? 0 : 0x80)); + } else if (i < n - 1) { + is_edge_case = (*b == 0); + } else { + is_edge_case = (*b == (little_endian ? 0x80 : 0)); + } + } + if (is_edge_case) { + res = n; + } + } + else { + /* Positive values with the MSB set do not require an + * additional bit when the caller's intent is to treat them + * as unsigned. */ + unsigned char *b = (unsigned char *)buffer; + if (b[little_endian ? n - 1 : 0] & 0x80) { + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + res = n; + } else { + res = n + 1; + } + } + } } } @@ -1222,38 +1277,41 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) PyObject * -PyLong_FromNativeBytes(const void* buffer, size_t n, int endianness) +PyLong_FromNativeBytes(const void* buffer, size_t n, int flags) { if (!buffer) { PyErr_BadInternalCall(); return NULL; } - int little_endian = endianness; + int little_endian = flags; if (_resolve_endianness(&little_endian) < 0) { return NULL; } - return _PyLong_FromByteArray((const unsigned char *)buffer, n, - little_endian, 1); + return _PyLong_FromByteArray( + (const unsigned char *)buffer, + n, + little_endian, + (flags == -1 || !(flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) ? 1 : 0 + ); } PyObject * -PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n, int endianness) +PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n, int flags) { if (!buffer) { PyErr_BadInternalCall(); return NULL; } - int little_endian = endianness; + int little_endian = flags; if (_resolve_endianness(&little_endian) < 0) { return NULL; } - return _PyLong_FromByteArray((const unsigned char *)buffer, n, - little_endian, 0); + return _PyLong_FromByteArray((const unsigned char *)buffer, n, little_endian, 0); } From 4d4a6f1b6aea6dae131ac116f1735a38c3e32cd1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 5 Apr 2024 17:00:29 +0200 Subject: [PATCH 095/143] gh-116608: Ignore UTF-16 BOM in importlib.resources._functional tests (GH-117569) gh-116609: Ignore UTF-16 BOM in importlib.resources._functional tests To test the `errors` argument, we read a UTF-16 file as UTF-8 with "backslashreplace" error handling. However, the utf-16 codec adds an endian-specific byte-order mark, so on big-endian machines the expectation doesn't match the test file (which was saved on a little-endian machine). Use endswith to ignore the BOM. --- .../test_importlib/resources/test_functional.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py index fd02fc7c0e7b15..d60a2bedd89798 100644 --- a/Lib/test/test_importlib/resources/test_functional.py +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -32,6 +32,12 @@ def _gen_resourcetxt_path_parts(self): with self.subTest(path_parts=path_parts): yield path_parts + def assertEndsWith(self, string, suffix): + """Assert that `string` ends with `suffix`. + + Used to ignore an architecture-specific UTF-16 byte-order mark.""" + self.assertEqual(string[-len(suffix):], suffix) + def test_read_text(self): self.assertEqual( resources.read_text(self.anchor01, 'utf-8.file'), @@ -65,12 +71,12 @@ def test_read_text(self): ), '\x00\x01\x02\x03', ) - self.assertEqual( + self.assertEndsWith( # ignore the BOM resources.read_text( self.anchor01, 'utf-16.file', errors='backslashreplace', ), - 'Hello, UTF-16 world!\n'.encode('utf-16').decode( + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( errors='backslashreplace', ), ) @@ -112,9 +118,9 @@ def test_open_text(self): self.anchor01, 'utf-16.file', errors='backslashreplace', ) as f: - self.assertEqual( + self.assertEndsWith( # ignore the BOM f.read(), - 'Hello, UTF-16 world!\n'.encode('utf-16').decode( + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( errors='backslashreplace', ), ) From 6150bb2412eb5ca3b330ccb9f0636949c7526a7f Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 5 Apr 2024 19:51:54 +0100 Subject: [PATCH 096/143] GH-77609: Add recurse_symlinks argument to `pathlib.Path.glob()` (#117311) Replace tri-state `follow_symlinks` with boolean `recurse_symlinks` argument. The new argument controls whether symlinks are followed when expanding recursive `**` wildcards. The possible argument values correspond as follows: follow_symlinks recurse_symlinks =============== ================ False N/A None False True True We therefore drop support for not following symlinks when expanding non-recursive pattern parts; it wasn't requested in the original issue, and it's a feature not found in any shells. This makes the API a easier to grok by eliminating `None` as an option. No news blurb as `follow_symlinks` was new in 3.13. --- Doc/library/pathlib.rst | 21 +++---- Doc/whatsnew/3.13.rst | 11 ++-- Lib/pathlib/__init__.py | 8 +-- Lib/pathlib/_abc.py | 20 +++---- Lib/test/test_pathlib/test_pathlib_abc.py | 69 +++-------------------- 5 files changed, 34 insertions(+), 95 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 9122df7a476632..f4ed479401f65c 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -985,7 +985,7 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 -.. method:: Path.glob(pattern, *, case_sensitive=None, follow_symlinks=None) +.. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) Glob the given relative *pattern* in the directory represented by this path, yielding all matching files (of any kind):: @@ -1013,14 +1013,9 @@ call fails (for example because the path doesn't exist). typically, case-sensitive on POSIX, and case-insensitive on Windows. Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - By default, or when the *follow_symlinks* keyword-only argument is set to - ``None``, this method follows symlinks except when expanding "``**``" - wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or - ``False`` to treat all symlinks as files. - - .. tip:: - Set *follow_symlinks* to ``True`` or ``False`` to improve performance - of recursive globbing. + By default, or when the *recurse_symlinks* keyword-only argument is set to + ``False``, this method follows symlinks except when expanding "``**``" + wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob @@ -1028,13 +1023,13 @@ call fails (for example because the path doesn't exist). The *case_sensitive* parameter was added. .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. + The *recurse_symlinks* parameter was added. .. versionchanged:: 3.13 The *pattern* parameter accepts a :term:`path-like object`. -.. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) +.. method:: Path.rglob(pattern, *, case_sensitive=None, recurse_symlinks=False) Glob the given relative *pattern* recursively. This is like calling :func:`Path.glob` with "``**/``" added in front of the *pattern*. @@ -1048,7 +1043,7 @@ call fails (for example because the path doesn't exist). The *case_sensitive* parameter was added. .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. + The *recurse_symlinks* parameter was added. .. versionchanged:: 3.13 The *pattern* parameter accepts a :term:`path-like object`. @@ -1675,7 +1670,7 @@ The patterns accepted and results generated by :meth:`Path.glob` and passing ``recursive=True`` to :func:`glob.glob`. 3. "``**``" pattern components do not follow symlinks by default in pathlib. This behaviour has no equivalent in :func:`glob.glob`, but you can pass - ``follow_symlinks=True`` to :meth:`Path.glob` for compatible behaviour. + ``recurse_symlinks=True`` to :meth:`Path.glob` for compatible behaviour. 4. Like all :class:`PurePath` and :class:`Path` objects, the values returned from :meth:`Path.glob` and :meth:`Path.rglob` don't include trailing slashes. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 99a9545dd4e586..e31f0c52d4c5f5 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -559,12 +559,15 @@ pathlib implementation of :mod:`os.path` used for low-level path parsing and joining: either ``posixpath`` or ``ntpath``. -* Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.glob`, - :meth:`~pathlib.Path.rglob`, :meth:`~pathlib.Path.is_file`, +* Add *recurse_symlinks* keyword-only argument to :meth:`pathlib.Path.glob` + and :meth:`~pathlib.Path.rglob`. + (Contributed by Barney Gale in :gh:`77609`). + +* Add *follow_symlinks* keyword-only argument to :meth:`~pathlib.Path.is_file`, :meth:`~pathlib.Path.is_dir`, :meth:`~pathlib.Path.owner`, :meth:`~pathlib.Path.group`. - (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`, and - Kamil Turek in :gh:`107962`). + (Contributed by Barney Gale in :gh:`105793`, and Kamil Turek in + :gh:`107962`). * Return files and directories from :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` when given a pattern that ends with "``**``". In diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 6cccfb864e8206..747000f1a43475 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -619,7 +619,7 @@ def _make_child_relpath(self, name): path._tail_cached = tail + [name] return path - def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ @@ -627,9 +627,9 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): if not isinstance(pattern, PurePath): pattern = self.with_segments(pattern) return _abc.PathBase.glob( - self, pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) + self, pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) - def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): """Recursively yield all existing files (of any kind, including directories) matching the given relative pattern, anywhere in this subtree. @@ -639,7 +639,7 @@ def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): pattern = self.with_segments(pattern) pattern = '**' / pattern return _abc.PathBase.glob( - self, pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) + self, pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) def walk(self, top_down=True, on_error=None, follow_symlinks=False): """Walk the directory tree from this directory, similar to os.walk().""" diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 932020e6d0866c..ca38a51d072cfb 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -66,10 +66,8 @@ def _select_special(paths, part): yield path._make_child_relpath(part) -def _select_children(parent_paths, dir_only, follow_symlinks, match): +def _select_children(parent_paths, dir_only, match): """Yield direct children of given paths, filtering by name and type.""" - if follow_symlinks is None: - follow_symlinks = True for parent_path in parent_paths: try: # We must close the scandir() object before proceeding to @@ -82,7 +80,7 @@ def _select_children(parent_paths, dir_only, follow_symlinks, match): for entry in entries: if dir_only: try: - if not entry.is_dir(follow_symlinks=follow_symlinks): + if not entry.is_dir(): continue except OSError: continue @@ -96,8 +94,6 @@ def _select_recursive(parent_paths, dir_only, follow_symlinks, match): """Yield given paths and all their children, recursively, filtering by string and type. """ - if follow_symlinks is None: - follow_symlinks = False for parent_path in parent_paths: if match is not None: # If we're filtering paths through a regex, record the length of @@ -789,7 +785,7 @@ def _make_child_direntry(self, entry): def _make_child_relpath(self, name): return self.joinpath(name) - def glob(self, pattern, *, case_sensitive=None, follow_symlinks=True): + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ @@ -818,7 +814,7 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=True): # Consume following non-special components, provided we're # treating symlinks consistently. Each component is joined # onto 'part', which is used to generate an re.Pattern object. - if follow_symlinks is not None: + if recurse_symlinks: while stack and stack[-1] not in specials: part += sep + stack.pop() @@ -827,7 +823,7 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=True): match = _compile_pattern(part, sep, case_sensitive) if part != '**' else None # Recursively walk directories, filtering by type and regex. - paths = _select_recursive(paths, bool(stack), follow_symlinks, match) + paths = _select_recursive(paths, bool(stack), recurse_symlinks, match) # De-duplicate if we've already seen a '**' component. if deduplicate_paths: @@ -843,10 +839,10 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=True): match = _compile_pattern(part, sep, case_sensitive) if part != '*' else None # Iterate over directories' children filtering by type and regex. - paths = _select_children(paths, bool(stack), follow_symlinks, match) + paths = _select_children(paths, bool(stack), match) return paths - def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=True): + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): """Recursively yield all existing files (of any kind, including directories) matching the given relative pattern, anywhere in this subtree. @@ -854,7 +850,7 @@ def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=True): if not isinstance(pattern, PurePathBase): pattern = self.with_segments(pattern) pattern = '**' / pattern - return self.glob(pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) def walk(self, top_down=True, on_error=None, follow_symlinks=False): """Walk the directory tree from this directory, similar to os.walk().""" diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index a7e35a3e1fc7da..336115cf0fead2 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1776,9 +1776,9 @@ def _check(path, pattern, case_sensitive, expected): _check(path, "dirb/file*", False, ["dirB/fileB"]) @needs_symlinks - def test_glob_follow_symlinks_common(self): + def test_glob_recurse_symlinks_common(self): def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=True) + actual = {path for path in path.glob(glob, recurse_symlinks=True) if path.parts.count("linkD") <= 1} # exclude symlink loop. self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls @@ -1812,39 +1812,9 @@ def _check(path, glob, expected): _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "*/dirD/**/", ["dirC/dirD/"]) - @needs_symlinks - def test_glob_no_follow_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"]) - _check(p, "dir*/*/..", ["dirC/dirD/.."]) - _check(p, "dir*/**", [ - "dirA/", "dirA/linkC", - "dirB/", "dirB/fileB", "dirB/linkD", - "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", - "dirE/"]) - _check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "dir*/*/**/", ["dirC/dirD/"]) - _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**", ["dirC/dirD/../dirD/", "dirC/dirD/../dirD/fileD"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) - _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "*/dirD/**/", ["dirC/dirD/"]) - - def test_rglob_follow_symlinks_none(self): + def test_rglob_recurse_symlinks_false(self): def _check(path, glob, expected): - actual = set(path.rglob(glob, follow_symlinks=None)) + actual = set(path.rglob(glob, recurse_symlinks=False)) self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls p = P(self.base) @@ -1901,9 +1871,9 @@ def test_rglob_windows(self): self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) @needs_symlinks - def test_rglob_follow_symlinks_common(self): + def test_rglob_recurse_symlinks_common(self): def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=True) + actual = {path for path in path.rglob(glob, recurse_symlinks=True) if path.parts.count("linkD") <= 1} # exclude symlink loop. self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls @@ -1932,37 +1902,12 @@ def _check(path, glob, expected): _check(p, "*.txt", ["dirC/novel.txt"]) _check(p, "*.*", ["dirC/novel.txt"]) - @needs_symlinks - def test_rglob_no_follow_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", ["dirB/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", ]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "", ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) - - p = P(self.base, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD/"]) - _check(p, "", ["dirC/", "dirC/dirD/"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - @needs_symlinks def test_rglob_symlink_loop(self): # Don't get fooled by symlink loops (Issue #26012). P = self.cls p = P(self.base) - given = set(p.rglob('*', follow_symlinks=None)) + given = set(p.rglob('*', recurse_symlinks=False)) expect = {'brokenLink', 'dirA', 'dirA/linkC', 'dirB', 'dirB/fileB', 'dirB/linkD', From 1d3225ae056245da75e4a443ccafcc8f4f982cf2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 5 Apr 2024 21:57:36 +0100 Subject: [PATCH 097/143] gh-116622: Test updates for Android (#117299) - re-enable test_fcntl_64_bit on Linux aarch64, but disable it on all Android ABIs - use support.setswitchinterval in all relevant tests - skip test_fma_zero_result on Android x86_64 - accept EACCES when calling os.get_terminal_size on Android --- Lib/test/_test_multiprocessing.py | 2 +- Lib/test/test_concurrent_futures/test_wait.py | 2 +- Lib/test/test_fcntl.py | 6 +++--- Lib/test/test_gc.py | 3 ++- Lib/test/test_importlib/test_threaded_import.py | 3 ++- Lib/test/test_math.py | 10 +++++----- Lib/test/test_os.py | 7 ++++++- Lib/test/test_threading.py | 2 +- 8 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b63b567bbcad08..a74b61013c4848 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4662,7 +4662,7 @@ def make_finalizers(): old_interval = sys.getswitchinterval() old_threshold = gc.get_threshold() try: - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) gc.set_threshold(5, 5, 5) threads = [threading.Thread(target=run_finalizers), threading.Thread(target=make_finalizers)] diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index ff486202092c81..108cf54bf79e6f 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -142,7 +142,7 @@ def test_pending_calls_race(self): def future_func(): event.wait() oldswitchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) try: fs = {self.executor.submit(future_func) for i in range(100)} event.set() diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 84c0e5bc4800a1..bb784cbe2ce036 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -131,9 +131,9 @@ def test_fcntl_bad_file_overflow(self): fcntl.fcntl(BadFile(INT_MIN - 1), fcntl.F_SETFL, os.O_NONBLOCK) @unittest.skipIf( - platform.machine().startswith(("arm", "aarch")) - and platform.system() in ("Linux", "Android"), - "ARM Linux returns EINVAL for F_NOTIFY DN_MULTISHOT") + (platform.machine().startswith("arm") and platform.system() == "Linux") + or platform.system() == "Android", + "this platform returns EINVAL for F_NOTIFY DN_MULTISHOT") def test_fcntl_64_bit(self): # Issue #1309352: fcntl shouldn't fail when the third arg fits in a # C 'long' but not in a C 'int'. diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 8a748cb55538e8..71c7fb0edebaa5 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,5 +1,6 @@ import unittest import unittest.mock +from test import support from test.support import (verbose, refcount_test, cpython_only, requires_subprocess, requires_gil_enabled) @@ -470,7 +471,7 @@ def run_thread(): make_nested() old_switchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-5) + support.setswitchinterval(1e-5) try: exit = [] threads = [] diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 5072be86cfd112..9af1e4d505c66e 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -13,6 +13,7 @@ import shutil import threading import unittest +from test import support from test.support import verbose from test.support.import_helper import forget, mock_register_at_fork from test.support.os_helper import (TESTFN, unlink, rmtree) @@ -260,7 +261,7 @@ def setUpModule(): try: old_switchinterval = sys.getswitchinterval() unittest.addModuleCleanup(sys.setswitchinterval, old_switchinterval) - sys.setswitchinterval(1e-5) + support.setswitchinterval(1e-5) except AttributeError: pass diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index aaa3b16d33fb7d..0e4dbc0b64a439 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2691,12 +2691,12 @@ def test_fma_infinities(self): self.assertEqual(math.fma(-b, -math.inf, c), math.inf) self.assertEqual(math.fma(-b, math.inf, c), -math.inf) - # gh-73468: On WASI and FreeBSD, libc fma() doesn't implement IEE 754-2008 + # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 # properly: it doesn't use the right sign when the result is zero. - @unittest.skipIf(support.is_wasi, - "WASI fma() doesn't implement IEE 754-2008 properly") - @unittest.skipIf(sys.platform.startswith('freebsd'), - "FreeBSD fma() doesn't implement IEE 754-2008 properly") + @unittest.skipIf( + sys.platform.startswith(("freebsd", "wasi")) + or (sys.platform == "android" and platform.machine() == "x86_64"), + f"this platform doesn't implement IEE 754-2008 properly") def test_fma_zero_result(self): nonnegative_finites = [0.0, 1e-300, 2.3, 1e300] diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 094ac13a915e2d..6a34f48f7873ee 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3934,7 +3934,12 @@ def test_does_not_crash(self): try: size = os.get_terminal_size() except OSError as e: - if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY): + known_errnos = [errno.EINVAL, errno.ENOTTY] + if sys.platform == "android": + # The Android testbed redirects the native stdout to a pipe, + # which returns a different error code. + known_errnos.append(errno.EACCES) + if sys.platform == "win32" or e.errno in known_errnos: # Under win32 a generic OSError can be thrown if the # handle cannot be retrieved self.skipTest("failed to query terminal size") diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 84a946477818df..a7701fa285aee2 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -515,7 +515,7 @@ def test_enumerate_after_join(self): old_interval = sys.getswitchinterval() try: for i in range(1, 100): - sys.setswitchinterval(i * 0.0002) + support.setswitchinterval(i * 0.0002) t = threading.Thread(target=lambda: None) t.start() t.join() From df4d84c3cdca572f1be8f5dc5ef8ead5351b51fb Mon Sep 17 00:00:00 2001 From: Laurie O Date: Sun, 7 Apr 2024 00:27:13 +1000 Subject: [PATCH 098/143] gh-96471: Add asyncio queue shutdown (#104228) Co-authored-by: Duprat --- Doc/library/asyncio-queue.rst | 31 +++ Doc/whatsnew/3.13.rst | 4 + Lib/asyncio/queues.py | 68 +++++- Lib/test/test_asyncio/test_queues.py | 199 ++++++++++++++++++ ...3-05-06-05-00-42.gh-issue-96471.S3X5I-.rst | 2 + 5 files changed, 301 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-06-05-00-42.gh-issue-96471.S3X5I-.rst diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d86fbc21351e2d..030d4310942d7a 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -62,6 +62,9 @@ Queue Remove and return an item from the queue. If queue is empty, wait until an item is available. + Raises :exc:`QueueShutDown` if the queue has been shut down and + is empty, or if the queue has been shut down immediately. + .. method:: get_nowait() Return an item if one is immediately available, else raise @@ -82,6 +85,8 @@ Queue Put an item into the queue. If the queue is full, wait until a free slot is available before adding the item. + Raises :exc:`QueueShutDown` if the queue has been shut down. + .. method:: put_nowait(item) Put an item into the queue without blocking. @@ -92,6 +97,21 @@ Queue Return the number of items in the queue. + .. method:: shutdown(immediate=False) + + Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` + raise :exc:`QueueShutDown`. + + By default, :meth:`~Queue.get` on a shut down queue will only + raise once the queue is empty. Set *immediate* to true to make + :meth:`~Queue.get` raise immediately instead. + + All blocked callers of :meth:`~Queue.put` will be unblocked. If + *immediate* is true, also unblock callers of :meth:`~Queue.get` + and :meth:`~Queue.join`. + + .. versionadded:: 3.13 + .. method:: task_done() Indicate that a formerly enqueued task is complete. @@ -105,6 +125,9 @@ Queue call was received for every item that had been :meth:`~Queue.put` into the queue). + ``shutdown(immediate=True)`` calls :meth:`task_done` for each + remaining item in the queue. + Raises :exc:`ValueError` if called more times than there were items placed in the queue. @@ -145,6 +168,14 @@ Exceptions on a queue that has reached its *maxsize*. +.. exception:: QueueShutDown + + Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is + called on a queue which has been shut down. + + .. versionadded:: 3.13 + + Examples ======== diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e31f0c52d4c5f5..c785d4cfa8fdc3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -296,6 +296,10 @@ asyncio with the tasks being completed. (Contributed by Justin Arthur in :gh:`77714`.) +* Add :meth:`asyncio.Queue.shutdown` (along with + :exc:`asyncio.QueueShutDown`) for queue termination. + (Contributed by Laurie Opperman in :gh:`104228`.) + base64 ------ diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index a9656a6df561ba..b8156704b8fc23 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -1,4 +1,11 @@ -__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty') +__all__ = ( + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'QueueFull', + 'QueueEmpty', + 'QueueShutDown', +) import collections import heapq @@ -18,6 +25,11 @@ class QueueFull(Exception): pass +class QueueShutDown(Exception): + """Raised when putting on to or getting from a shut-down Queue.""" + pass + + class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. @@ -41,6 +53,7 @@ def __init__(self, maxsize=0): self._finished = locks.Event() self._finished.set() self._init(maxsize) + self._is_shutdown = False # These three are overridable in subclasses. @@ -81,6 +94,8 @@ def _format(self): result += f' _putters[{len(self._putters)}]' if self._unfinished_tasks: result += f' tasks={self._unfinished_tasks}' + if self._is_shutdown: + result += ' shutdown' return result def qsize(self): @@ -112,8 +127,12 @@ async def put(self, item): Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. + + Raises QueueShutDown if the queue has been shut down. """ while self.full(): + if self._is_shutdown: + raise QueueShutDown putter = self._get_loop().create_future() self._putters.append(putter) try: @@ -125,7 +144,7 @@ async def put(self, item): self._putters.remove(putter) except ValueError: # The putter could be removed from self._putters by a - # previous get_nowait call. + # previous get_nowait call or a shutdown call. pass if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take @@ -138,7 +157,11 @@ def put_nowait(self, item): """Put an item into the queue without blocking. If no free slot is immediately available, raise QueueFull. + + Raises QueueShutDown if the queue has been shut down. """ + if self._is_shutdown: + raise QueueShutDown if self.full(): raise QueueFull self._put(item) @@ -150,8 +173,13 @@ async def get(self): """Remove and return an item from the queue. If queue is empty, wait until an item is available. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ while self.empty(): + if self._is_shutdown and self.empty(): + raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) try: @@ -163,7 +191,7 @@ async def get(self): self._getters.remove(getter) except ValueError: # The getter could be removed from self._getters by a - # previous put_nowait call. + # previous put_nowait call, or a shutdown call. pass if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take @@ -176,8 +204,13 @@ def get_nowait(self): """Remove and return an item from the queue. Return an item if one is immediately available, else raise QueueEmpty. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ if self.empty(): + if self._is_shutdown: + raise QueueShutDown raise QueueEmpty item = self._get() self._wakeup_next(self._putters) @@ -194,6 +227,9 @@ def task_done(self): been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises ValueError if called more times than there were items placed in the queue. """ @@ -214,6 +250,32 @@ async def join(self): if self._unfinished_tasks > 0: await self._finished.wait() + def shutdown(self, immediate=False): + """Shut-down the queue, making queue gets and puts raise QueueShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() will be unblocked, and also get() + and join() if 'immediate'. + """ + self._is_shutdown = True + if immediate: + while not self.empty(): + self._get() + if self._unfinished_tasks > 0: + self._unfinished_tasks -= 1 + if self._unfinished_tasks == 0: + self._finished.set() + while self._getters: + getter = self._getters.popleft() + if not getter.done(): + getter.set_result(None) + while self._putters: + putter = self._putters.popleft() + if not putter.done(): + putter.set_result(None) + class PriorityQueue(Queue): """A subclass of Queue; retrieves entries in priority order (lowest first). diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 2d058ccf6a8c72..5019e9a293525d 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -522,5 +522,204 @@ class PriorityQueueJoinTests(_QueueJoinTestMixin, unittest.IsolatedAsyncioTestCa q_class = asyncio.PriorityQueue +class _QueueShutdownTestMixin: + q_class = None + + def assertRaisesShutdown(self, msg="Didn't appear to shut-down queue"): + return self.assertRaises(asyncio.QueueShutDown, msg=msg) + + async def test_format(self): + q = self.q_class() + q.shutdown() + self.assertEqual(q._format(), 'maxsize=0 shutdown') + + async def test_shutdown_empty(self): + # Test shutting down an empty queue + + # Setup empty queue, and join() and get() tasks + q = self.q_class() + loop = asyncio.get_running_loop() + get_task = loop.create_task(q.get()) + await asyncio.sleep(0) # want get task pending before shutdown + + # Perform shut-down + q.shutdown(immediate=False) # unfinished tasks: 0 -> 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task successfully finishes + await q.join() + + # Ensure get() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(get_task.done()) + with self.assertRaisesShutdown(): + await get_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + async def test_shutdown_nonempty(self): + # Test shutting down a non-empty queue + + # Setup full queue with 1 item, and join() and put() tasks + q = self.q_class(maxsize=1) + loop = asyncio.get_running_loop() + + q.put_nowait("data") + join_task = loop.create_task(q.join()) + put_task = loop.create_task(q.put("data2")) + + # Ensure put() task is not finished + await asyncio.sleep(0) + self.assertFalse(put_task.done()) + + # Perform shut-down + q.shutdown(immediate=False) # unfinished tasks: 1 -> 1 + + self.assertEqual(q.qsize(), 1) + + # Ensure put() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(put_task.done()) + with self.assertRaisesShutdown(): + await put_task + + # Ensure get() succeeds on enqueued item + self.assertEqual(await q.get(), "data") + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task, and join() task succeeds + q.task_done() + + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate(self): + # Test immediately shutting down a queue + + # Setup queue with 1 item, and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + join_task = loop.create_task(q.join()) + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 1 -> 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there are no unfinished tasks + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate_with_unfinished(self): + # Test immediately shutting down a queue with unfinished tasks + + # Setup queue with 2 items (1 retrieved), and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + q.put_nowait("data") + join_task = loop.create_task(q.join()) + self.assertEqual(await q.get(), "data") + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 2 -> 1 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task + q.task_done() + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + +class QueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.Queue + + +class LifoQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.LifoQueue + + +class PriorityQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.PriorityQueue + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-05-06-05-00-42.gh-issue-96471.S3X5I-.rst b/Misc/NEWS.d/next/Library/2023-05-06-05-00-42.gh-issue-96471.S3X5I-.rst new file mode 100644 index 00000000000000..128a85d3d73ddf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-06-05-00-42.gh-issue-96471.S3X5I-.rst @@ -0,0 +1,2 @@ +Add :py:class:`asyncio.Queue` termination with +:py:meth:`~asyncio.Queue.shutdown` method. From 62aeb0ee69b06091396398de56dcb755ca3b9dc9 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 6 Apr 2024 08:26:43 -0700 Subject: [PATCH 099/143] GH-117512: Allow 64-bit JIT operands on 32-bit platforms (GH-117527) --- Python/jit.c | 62 +++++++++++++++++++++++------------------- Tools/jit/_stencils.py | 5 +++- Tools/jit/_writer.py | 4 +-- Tools/jit/template.c | 7 +++++ 4 files changed, 47 insertions(+), 31 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index 03bcf1142715f3..8782adb847cfd6 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -149,12 +149,12 @@ set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, // Fill all of stencil's holes in the memory pointed to by base, using the // values in patches. static void -patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) +patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) { - for (uint64_t i = 0; i < stencil->holes_size; i++) { + for (size_t i = 0; i < stencil->holes_size; i++) { const Hole *hole = &stencil->holes[i]; unsigned char *location = base + hole->offset; - uint64_t value = patches[hole->value] + (uint64_t)hole->symbol + hole->addend; + uint64_t value = patches[hole->value] + (uintptr_t)hole->symbol + hole->addend; uint8_t *loc8 = (uint8_t *)location; uint32_t *loc32 = (uint32_t *)location; uint64_t *loc64 = (uint64_t *)location; @@ -228,7 +228,7 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) case HoleKind_X86_64_RELOC_SIGNED: case HoleKind_X86_64_RELOC_BRANCH: // 32-bit relative address. - value -= (uint64_t)location; + value -= (uintptr_t)location; // Check that we're not out of range of 32 signed bits: assert((int64_t)value >= -(1LL << 31)); assert((int64_t)value < (1LL << 31)); @@ -239,7 +239,7 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) case HoleKind_R_AARCH64_JUMP26: // 28-bit relative branch. assert(IS_AARCH64_BRANCH(*loc32)); - value -= (uint64_t)location; + value -= (uintptr_t)location; // Check that we're not out of range of 28 signed bits: assert((int64_t)value >= -(1 << 27)); assert((int64_t)value < (1 << 27)); @@ -313,7 +313,7 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) i++; continue; } - relaxed = (uint64_t)value - (uint64_t)location; + relaxed = value - (uintptr_t)location; if ((relaxed & 0x3) == 0 && (int64_t)relaxed >= -(1L << 19) && (int64_t)relaxed < (1L << 19)) @@ -328,7 +328,7 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) // Fall through... case HoleKind_ARM64_RELOC_PAGE21: // Number of pages between this page and the value's page: - value = (value >> 12) - ((uint64_t)location >> 12); + value = (value >> 12) - ((uintptr_t)location >> 12); // Check that we're not out of range of 21 signed bits: assert((int64_t)value >= -(1 << 20)); assert((int64_t)value < (1 << 20)); @@ -363,14 +363,14 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) } static void -copy_and_patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) +copy_and_patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) { memcpy(base, stencil->body, stencil->body_size); patch(base, stencil, patches); } static void -emit(const StencilGroup *group, uint64_t patches[]) +emit(const StencilGroup *group, uintptr_t patches[]) { copy_and_patch((unsigned char *)patches[HoleValue_DATA], &group->data, patches); copy_and_patch((unsigned char *)patches[HoleValue_CODE], &group->code, patches); @@ -381,9 +381,9 @@ int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length) { // Loop once to find the total compiled size: - uint32_t instruction_starts[UOP_MAX_TRACE_LENGTH]; - uint32_t code_size = 0; - uint32_t data_size = 0; + size_t instruction_starts[UOP_MAX_TRACE_LENGTH]; + size_t code_size = 0; + size_t data_size = 0; for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; @@ -409,14 +409,20 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; - // Think of patches as a dictionary mapping HoleValue to uint64_t: - uint64_t patches[] = GET_PATCHES(); - patches[HoleValue_CODE] = (uint64_t)code; - patches[HoleValue_CONTINUE] = (uint64_t)code + group->code.body_size; - patches[HoleValue_DATA] = (uint64_t)data; - patches[HoleValue_EXECUTOR] = (uint64_t)executor; + // Think of patches as a dictionary mapping HoleValue to uintptr_t: + uintptr_t patches[] = GET_PATCHES(); + patches[HoleValue_CODE] = (uintptr_t)code; + patches[HoleValue_CONTINUE] = (uintptr_t)code + group->code.body_size; + patches[HoleValue_DATA] = (uintptr_t)data; + patches[HoleValue_EXECUTOR] = (uintptr_t)executor; patches[HoleValue_OPARG] = instruction->oparg; + #if SIZEOF_VOID_P == 8 patches[HoleValue_OPERAND] = instruction->operand; + #else + assert(SIZEOF_VOID_P == 4); + patches[HoleValue_OPERAND_HI] = instruction->operand >> 32; + patches[HoleValue_OPERAND_LO] = instruction->operand & UINT32_MAX; + #endif switch (instruction->format) { case UOP_FORMAT_TARGET: patches[HoleValue_TARGET] = instruction->target; @@ -425,21 +431,21 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size assert(instruction->exit_index < executor->exit_count); patches[HoleValue_EXIT_INDEX] = instruction->exit_index; if (instruction->error_target < length) { - patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target]; + patches[HoleValue_ERROR_TARGET] = (uintptr_t)memory + instruction_starts[instruction->error_target]; } break; case UOP_FORMAT_JUMP: assert(instruction->jump_target < length); - patches[HoleValue_JUMP_TARGET] = (uint64_t)memory + instruction_starts[instruction->jump_target]; + patches[HoleValue_JUMP_TARGET] = (uintptr_t)memory + instruction_starts[instruction->jump_target]; if (instruction->error_target < length) { - patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target]; + patches[HoleValue_ERROR_TARGET] = (uintptr_t)memory + instruction_starts[instruction->error_target]; } break; default: assert(0); Py_FatalError("Illegal instruction format"); } - patches[HoleValue_TOP] = (uint64_t)memory + instruction_starts[1]; + patches[HoleValue_TOP] = (uintptr_t)memory + instruction_starts[1]; patches[HoleValue_ZERO] = 0; emit(group, patches); code += group->code.body_size; @@ -447,12 +453,12 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size } // Protect against accidental buffer overrun into data: const StencilGroup *group = &stencil_groups[_FATAL_ERROR]; - uint64_t patches[] = GET_PATCHES(); - patches[HoleValue_CODE] = (uint64_t)code; - patches[HoleValue_CONTINUE] = (uint64_t)code; - patches[HoleValue_DATA] = (uint64_t)data; - patches[HoleValue_EXECUTOR] = (uint64_t)executor; - patches[HoleValue_TOP] = (uint64_t)code; + uintptr_t patches[] = GET_PATCHES(); + patches[HoleValue_CODE] = (uintptr_t)code; + patches[HoleValue_CONTINUE] = (uintptr_t)code; + patches[HoleValue_DATA] = (uintptr_t)data; + patches[HoleValue_EXECUTOR] = (uintptr_t)executor; + patches[HoleValue_TOP] = (uintptr_t)code; patches[HoleValue_ZERO] = 0; emit(group, patches); code += group->code.body_size; diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 601ea0b70701a5..243bb3dd134f70 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -27,8 +27,11 @@ class HoleValue(enum.Enum): GOT = enum.auto() # The current uop's oparg (exposed as _JIT_OPARG): OPARG = enum.auto() - # The current uop's operand (exposed as _JIT_OPERAND): + # The current uop's operand on 64-bit platforms (exposed as _JIT_OPERAND): OPERAND = enum.auto() + # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI and _JIT_OPERAND_LO): + OPERAND_HI = enum.auto() + OPERAND_LO = enum.auto() # The current uop's target (exposed as _JIT_TARGET): TARGET = enum.auto() # The base address of the machine code for the jump target (exposed as _JIT_JUMP_TARGET): diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 8a2a42e75cfb9b..cbc1ed2fa6543a 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -17,7 +17,7 @@ def _dump_header() -> typing.Iterator[str]: yield "} HoleValue;" yield "" yield "typedef struct {" - yield " const uint64_t offset;" + yield " const size_t offset;" yield " const HoleKind kind;" yield " const HoleValue value;" yield " const void *symbol;" @@ -58,7 +58,7 @@ def _dump_footer(opnames: typing.Iterable[str]) -> typing.Iterator[str]: yield "" yield "#define GET_PATCHES() { \\" for value in _stencils.HoleValue: - yield f" [HoleValue_{value.name}] = (uint64_t)0xBADBADBADBADBADB, \\" + yield f" [HoleValue_{value.name}] = (uintptr_t)0xBADBADBADBADBADB, \\" yield "}" diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 2300bd0f1f31ec..b195aff377b3b5 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -88,7 +88,14 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * int uopcode = _JIT_OPCODE; // Other stuff we need handy: PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) +#if SIZEOF_VOID_P == 8 PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) +#else + assert(SIZEOF_VOID_P == 4); + PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI) + PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO) + uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo; +#endif PATCH_VALUE(uint32_t, _target, _JIT_TARGET) PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX) From 733e56ef9656dd79055acc2a3cecaf6054a45b6c Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Sun, 7 Apr 2024 11:00:08 +0200 Subject: [PATCH 100/143] gh-117584: Raise TypeError for non-paths in posixpath.relpath() (GH-117585) --- Lib/posixpath.py | 2 +- Lib/test/test_posixpath.py | 1 + .../2024-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 0e8bb5ab10d916..b7fbdff20cac99 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -502,10 +502,10 @@ def realpath(filename, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" + path = os.fspath(path) if not path: raise ValueError("no path specified") - path = os.fspath(path) if isinstance(path, bytes): curdir = b'.' sep = b'/' diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 807f985f7f4df7..ff78410738022d 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -650,6 +650,7 @@ def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") try: curdir = os.path.split(os.getcwd())[-1] + self.assertRaises(TypeError, posixpath.relpath, None) self.assertRaises(ValueError, posixpath.relpath, "") self.assertEqual(posixpath.relpath("a"), "a") self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst new file mode 100644 index 00000000000000..fd6a6097a89154 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst @@ -0,0 +1 @@ +Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath()`. From 375425abd17310480988c48fba57b01e8c979e07 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Mon, 8 Apr 2024 06:26:52 +0800 Subject: [PATCH 101/143] Cases generator: Remove type_prop and passthrough (#117614) --- Include/internal/pycore_uop_metadata.h | 44 +++++++++++----------- Tools/cases_generator/analyzer.py | 9 ----- Tools/cases_generator/generators_common.py | 2 - 3 files changed, 22 insertions(+), 33 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 111824a938f6cc..e5a99421c241e0 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -51,22 +51,22 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_BOOL] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, [_TO_BOOL_INT] = HAS_EXIT_FLAG, [_TO_BOOL_LIST] = HAS_EXIT_FLAG, [_TO_BOOL_NONE] = HAS_EXIT_FLAG, [_TO_BOOL_STR] = HAS_EXIT_FLAG, [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BOTH_INT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_INT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, - [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -129,23 +129,23 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, - [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG, [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, - [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG, [_LOAD_ATTR_CLASS_0] = 0, [_LOAD_ATTR_CLASS_1] = 0, [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, - [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = 0, [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -167,31 +167,31 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG, + [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG, [_ITER_NEXT_LIST] = 0, - [_ITER_CHECK_TUPLE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_ITER_CHECK_TUPLE] = HAS_DEOPT_FLAG, + [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_DEOPT_FLAG, [_ITER_NEXT_TUPLE] = 0, - [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG, + [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, + [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, - [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, - [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 4261378d459107..e38ab3c9047039 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -24,7 +24,6 @@ class Properties: has_free: bool side_exit: bool pure: bool - passthrough: bool tier: int | None = None oparg_and_1: bool = False const_oparg: int = -1 @@ -54,7 +53,6 @@ def from_list(properties: list["Properties"]) -> "Properties": has_free=any(p.has_free for p in properties), side_exit=any(p.side_exit for p in properties), pure=all(p.pure for p in properties), - passthrough=all(p.passthrough for p in properties), ) @property @@ -81,7 +79,6 @@ def infallible(self) -> bool: has_free=False, side_exit=False, pure=False, - passthrough=False, ) @@ -106,9 +103,6 @@ class StackItem: condition: str | None size: str peek: bool = False - type_prop: None | tuple[str, None | str] = field( - default_factory=lambda: None, init=True, compare=False, hash=False - ) def __str__(self) -> str: cond = f" if ({self.condition})" if self.condition else "" @@ -536,8 +530,6 @@ def compute_properties(op: parser.InstDef) -> Properties: ) error_with_pop = has_error_with_pop(op) error_without_pop = has_error_without_pop(op) - infallible = not error_with_pop and not error_without_pop - passthrough = stack_effect_only_peeks(op) and infallible return Properties( escapes=makes_escaping_api_call(op), error_with_pop=error_with_pop, @@ -557,7 +549,6 @@ def compute_properties(op: parser.InstDef) -> Properties: and not has_free, has_free=has_free, pure="pure" in op.annotations, - passthrough=passthrough, tier=tier_variable(op), ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 0addcf0ab570f6..cc9eb8a0e90eeb 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -234,8 +234,6 @@ def cflags(p: Properties) -> str: flags.append("HAS_ESCAPES_FLAG") if p.pure: flags.append("HAS_PURE_FLAG") - if p.passthrough: - flags.append("HAS_PASSTHROUGH_FLAG") if p.oparg_and_1: flags.append("HAS_OPARG_AND_1_FLAG") if flags: From fd3679025d9d0da7eb11f2810ed270c214926992 Mon Sep 17 00:00:00 2001 From: Anita Hammer <166057949+anitahammer@users.noreply.github.com> Date: Sun, 7 Apr 2024 23:58:57 +0100 Subject: [PATCH 102/143] Fix reference in code.rst (#117615) Co-authored-by: Alex Waygood --- Doc/library/code.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 091840781bd235..8cb604cf48ff0b 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -41,7 +41,7 @@ build applications which provide an interactive interpreter prompt. the :meth:`InteractiveConsole.raw_input` method, if provided. If *local* is provided, it is passed to the :class:`InteractiveConsole` constructor for use as the default namespace for the interpreter loop. If *local_exit* is provided, - it is passed to the :class:`InteractiveConsole` constructor. The :meth:`interact` + it is passed to the :class:`InteractiveConsole` constructor. The :meth:`~InteractiveConsole.interact` method of the instance is then run with *banner* and *exitmsg* passed as the banner and exit message to use, if provided. The console object is discarded after use. From 784623c63c45a4d13dfb04318c39fdb1ab790218 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 8 Apr 2024 08:58:19 +0900 Subject: [PATCH 103/143] gh-117594: Require cpu resource to test_search_anchor_at_beginning (gh-117595) --- Lib/test/test_re.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index b1ac22c28cf7c1..b8b50e8b3c2190 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,7 +1,7 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, check_disallow_instantiation, is_emscripten, is_wasi, - warnings_helper, SHORT_TIMEOUT, CPUStopwatch) + warnings_helper, SHORT_TIMEOUT, CPUStopwatch, requires_resource) import locale import re import string @@ -2282,6 +2282,9 @@ def test_bug_40736(self): with self.assertRaisesRegex(TypeError, "got 'type'"): re.search("x*", type) + # gh-117594: The test is not slow by itself, but it relies on + # the absolute computation time and can fail on very slow computers. + @requires_resource('cpu') def test_search_anchor_at_beginning(self): s = 'x'*10**7 with CPUStopwatch() as stopwatch: From a453f5ef9d0b89bd00488d3814c6f0a2886342b8 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 8 Apr 2024 09:35:48 +0300 Subject: [PATCH 104/143] gh-111726: Cleanup test files after running sqlite3 doctest (#117604) Remove all temporary databases in a dedicated 'testcleanup' step at the end of the file. --- Doc/library/sqlite3.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e76dc91bf2d875..b17ac19535df00 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2747,3 +2747,11 @@ regardless of the value of :attr:`~Connection.isolation_level`. .. _SQLite transaction behaviour: https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions + +.. testcleanup:: + + import os + os.remove("backup.db") + os.remove("dump.sql") + os.remove("example.db") + os.remove("tutorial.db") From e1eeb990bd169491075eeaea31481a4a96bdecbb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Apr 2024 09:51:20 +0200 Subject: [PATCH 105/143] gh-113317: Remove unused INVALID constant in Argument Clinic (#117624) --- Lib/test/test_clinic.py | 3 --- Tools/clinic/libclinic/function.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 9788ac0261fa49..a5887cdb56e3ca 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -4048,9 +4048,6 @@ def test_Class_repr(self): self.assertRegex(repr(cls), r"") def test_FunctionKind_repr(self): - self.assertEqual( - repr(FunctionKind.INVALID), "" - ) self.assertEqual( repr(FunctionKind.CLASS_METHOD), "" ) diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index 1563fdf9065b7e..572916bbe123b4 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -53,7 +53,6 @@ def __repr__(self) -> str: class FunctionKind(enum.Enum): - INVALID = enum.auto() CALLABLE = enum.auto() STATIC_METHOD = enum.auto() CLASS_METHOD = enum.auto() @@ -70,7 +69,6 @@ def __repr__(self) -> str: return f"" -INVALID: Final = FunctionKind.INVALID CALLABLE: Final = FunctionKind.CALLABLE STATIC_METHOD: Final = FunctionKind.STATIC_METHOD CLASS_METHOD: Final = FunctionKind.CLASS_METHOD From e338e1a4ec5e43a02447f4ec80320d7fc12b3ed4 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 8 Apr 2024 10:11:08 +0200 Subject: [PATCH 106/143] gh-111726: Remove some doctests from sqlite3 docs (#117623) * remove load extension doctest since we cannot skip it conditionally * remove sys.unraisablehook example; using unraisable hooks is not "an improved debug experience" --- Doc/library/sqlite3.rst | 43 +++++++---------------------------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index b17ac19535df00..9b857b068388d6 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -394,29 +394,11 @@ Module functions will get tracebacks from callbacks on :data:`sys.stderr`. Use ``False`` to disable the feature again. - Register an :func:`unraisable hook handler ` for an - improved debug experience: - - .. testsetup:: sqlite3.trace - - import sqlite3 + .. note:: - .. doctest:: sqlite3.trace - - >>> sqlite3.enable_callback_tracebacks(True) - >>> con = sqlite3.connect(":memory:") - >>> def evil_trace(stmt): - ... 5/0 - ... - >>> con.set_trace_callback(evil_trace) - >>> def debug(unraisable): - ... print(f"{unraisable.exc_value!r} in callback {unraisable.object.__name__}") - ... print(f"Error message: {unraisable.err_msg}") - >>> import sys - >>> sys.unraisablehook = debug - >>> cur = con.execute("SELECT 1") - ZeroDivisionError('division by zero') in callback evil_trace - Error message: None + Errors in user-defined function callbacks are logged as unraisable exceptions. + Use an :func:`unraisable hook handler ` for + introspection of the failed callback. .. function:: register_adapter(type, adapter, /) @@ -1068,13 +1050,10 @@ Connection objects .. versionchanged:: 3.10 Added the ``sqlite3.enable_load_extension`` auditing event. - .. testsetup:: sqlite3.loadext - - import sqlite3 - con = sqlite3.connect(":memory:") + .. We cannot doctest the load extension API, since there is no convenient + way to skip it. - .. testcode:: sqlite3.loadext - :skipif: True # not testable at the moment + .. code-block:: con.enable_load_extension(True) @@ -1098,14 +1077,6 @@ Connection objects for row in con.execute("SELECT rowid, name, ingredients FROM recipe WHERE name MATCH 'pie'"): print(row) - con.close() - - .. testoutput:: sqlite3.loadext - :hide: - - (2, 'broccoli pie', 'broccoli cheese onions flour') - (3, 'pumpkin pie', 'pumpkin sugar flour butter') - .. method:: load_extension(path, /, *, entrypoint=None) Load an SQLite extension from a shared library. From a7702663e3f7efc81f0b547f1f13ba64c4e5addc Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 8 Apr 2024 12:29:47 +0300 Subject: [PATCH 107/143] gh-111726: Explicitly close database connections in sqlite3 doctests (#111730) Co-authored-by: Erlend E. Aasland --- Doc/library/sqlite3.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9b857b068388d6..e6961821b639b9 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -16,6 +16,8 @@ src = sqlite3.connect(":memory:", isolation_level=None) dst = sqlite3.connect("tutorial.db", isolation_level=None) src.backup(dst) + src.close() + dst.close() del src, dst .. _sqlite3-intro: @@ -220,6 +222,7 @@ creating a new cursor, then querying the database: >>> title, year = res.fetchone() >>> print(f'The highest scoring Monty Python movie is {title!r}, released in {year}') The highest scoring Monty Python movie is 'Monty Python and the Holy Grail', released in 1975 + >>> new_con.close() You've now created an SQLite database using the :mod:`!sqlite3` module, inserted data and retrieved values from it in multiple ways. @@ -744,6 +747,7 @@ Connection objects >>> for row in con.execute("SELECT md5(?)", (b"foo",)): ... print(row) ('acbd18db4cc2f85cedef654fccc4a4d8',) + >>> con.close() .. versionchanged:: 3.13 @@ -890,6 +894,7 @@ Connection objects FROM test ORDER BY x """) print(cur.fetchall()) + con.close() .. testoutput:: :hide: @@ -1201,6 +1206,8 @@ Connection objects src = sqlite3.connect('example.db') dst = sqlite3.connect(':memory:') src.backup(dst) + dst.close() + src.close() .. versionadded:: 3.7 @@ -1267,6 +1274,10 @@ Connection objects >>> con.getlimit(sqlite3.SQLITE_LIMIT_ATTACHED) 1 + .. testcleanup:: sqlite3.limits + + con.close() + .. versionadded:: 3.11 .. _SQLite limit category: https://www.sqlite.org/c3ref/c_limit_attached.html @@ -1548,6 +1559,10 @@ Cursor objects # cur is an sqlite3.Cursor object cur.executemany("INSERT INTO data VALUES(?)", rows) + .. testcleanup:: sqlite3.cursor + + con.close() + .. note:: Any resulting rows are discarded, @@ -1653,6 +1668,7 @@ Cursor objects >>> cur = con.cursor() >>> cur.connection == con True + >>> con.close() .. attribute:: description @@ -1773,6 +1789,7 @@ Blob objects greeting = blob.read() print(greeting) # outputs "b'Hello, world!'" + con.close() .. testoutput:: :hide: @@ -2085,6 +2102,7 @@ Here's an example of both styles: params = (1972,) cur.execute("SELECT * FROM lang WHERE first_appeared = ?", params) print(cur.fetchall()) + con.close() .. testoutput:: :hide: @@ -2143,6 +2161,7 @@ The object passed to *protocol* will be of type :class:`PrepareProtocol`. cur.execute("SELECT ?", (Point(4.0, -3.2),)) print(cur.fetchone()[0]) + con.close() .. testoutput:: :hide: @@ -2173,6 +2192,7 @@ This function can then be registered using :func:`register_adapter`. cur.execute("SELECT ?", (Point(1.0, 2.5),)) print(cur.fetchone()[0]) + con.close() .. testoutput:: :hide: @@ -2257,6 +2277,8 @@ The following example illustrates the implicit and explicit approaches: cur.execute("INSERT INTO test(p) VALUES(?)", (p,)) cur.execute('SELECT p AS "p [point]" FROM test') print("with column names:", cur.fetchone()[0]) + cur.close() + con.close() .. testoutput:: :hide: @@ -2463,6 +2485,8 @@ Some useful URI tricks include: res = con2.execute("SELECT data FROM shared") assert res.fetchone() == (28,) + con1.close() + con2.close() More information about this feature, including a list of parameters, can be found in the `SQLite URI documentation`_. @@ -2509,6 +2533,7 @@ Queries now return :class:`!Row` objects: 'Earth' >>> row["RADIUS"] # Column names are case-insensitive. 6378 + >>> con.close() .. note:: @@ -2535,6 +2560,7 @@ Using it, queries now return a :class:`!dict` instead of a :class:`!tuple`: >>> for row in con.execute("SELECT 1 AS a, 2 AS b"): ... print(row) {'a': 1, 'b': 2} + >>> con.close() The following row factory returns a :term:`named tuple`: @@ -2561,6 +2587,7 @@ The following row factory returns a :term:`named tuple`: 1 >>> row.b # Attribute access. 2 + >>> con.close() With some adjustments, the above recipe can be adapted to use a :class:`~dataclasses.dataclass`, or any other custom class, From 9a12f5d1c19dee1f89684be776680aeaf117be5b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Apr 2024 14:19:43 +0200 Subject: [PATCH 108/143] gh-106023: Update What's New in 3.13: _PyObject_FastCall() (#117633) The function _PyObject_FastCall() was restored. --- Doc/whatsnew/3.13.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c785d4cfa8fdc3..9e40bf04c49bfa 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -2006,11 +2006,6 @@ Removed (Contributed by Victor Stinner in :gh:`105182`.) -* Remove private ``_PyObject_FastCall()`` function: - use ``PyObject_Vectorcall()`` which is available since Python 3.8 - (:pep:`590`). - (Contributed by Victor Stinner in :gh:`106023`.) - * Remove ``cpython/pytime.h`` header file: it only contained private functions. (Contributed by Victor Stinner in :gh:`106316`.) From ca62ffd1a5ef41401abceddfd171c76c68825a35 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 8 Apr 2024 14:45:25 +0200 Subject: [PATCH 109/143] gh-116303: Skip tests if C recursion limit is unavailable (GH-117368) The test suite fetches the C recursion limit from the _testcapi extension module. Test extension modules can be disabled using the --disable-test-modules configure option. --- Lib/test/list_tests.py | 4 ++-- Lib/test/mapping_tests.py | 4 ++-- Lib/test/support/__init__.py | 20 ++++++++------------ Lib/test/test_ast.py | 4 ++-- Lib/test/test_collections.py | 2 +- Lib/test/test_compile.py | 11 ++++++----- Lib/test/test_dict.py | 4 ++-- Lib/test/test_dictviews.py | 4 ++-- Lib/test/test_exception_group.py | 4 ++-- Lib/test/test_exceptions.py | 2 +- Lib/test/test_functools.py | 2 +- Lib/test/test_isinstance.py | 2 +- Lib/test/test_marshal.py | 2 +- Lib/test/test_sys_settrace.py | 2 +- 14 files changed, 32 insertions(+), 35 deletions(-) diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 26118e14bb97e0..89cd10f76a318e 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, Py_C_RECURSION_LIMIT +from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index b4cfce19a7174e..ed89a81a6ea685 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -1,7 +1,7 @@ # tests common to dict and UserDict import unittest import collections -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class BasicTestMappingProtocol(unittest.TestCase): @@ -624,7 +624,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2be9cd099a68d6..4bf2d7b5142da9 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -56,7 +56,7 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "Py_C_RECURSION_LIMIT", + "Py_DEBUG", "exceeds_recursion_limit", "get_c_recursion_limit", "skip_on_s390x", "without_optimizer", ] @@ -2490,22 +2490,18 @@ def adjust_int_max_str_digits(max_digits): sys.set_int_max_str_digits(current) -def _get_c_recursion_limit(): +def get_c_recursion_limit(): try: import _testcapi return _testcapi.Py_C_RECURSION_LIMIT - except (ImportError, AttributeError): - # Originally taken from Include/cpython/pystate.h . - if sys.platform == 'win32': - return 4000 - else: - return 10000 + except ImportError: + raise unittest.SkipTest('requires _testcapi') + -# The default C recursion limit. -Py_C_RECURSION_LIMIT = _get_c_recursion_limit() +def exceeds_recursion_limit(): + """For recursion tests, easily exceeds default recursion limit.""" + return get_c_recursion_limit() * 3 -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = Py_C_RECURSION_LIMIT * 3 #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 3929e4e00d59c2..5b47cdaafb092e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1153,9 +1153,9 @@ def next(self): @support.cpython_only def test_ast_recursion_limit(self): - fail_depth = support.EXCEEDS_RECURSION_LIMIT + fail_depth = support.exceeds_recursion_limit() crash_depth = 100_000 - success_depth = int(support.Py_C_RECURSION_LIMIT * 0.8) + success_depth = int(support.get_c_recursion_limit() * 0.8) if _testinternalcapi is not None: remaining = _testinternalcapi.get_c_recursion_remaining() success_depth = min(success_depth, remaining) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 1fb492ecebd668..955323cae88f92 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -542,7 +542,7 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) - n = support.EXCEEDS_RECURSION_LIMIT + n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) n = len(names) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 9d5f721806a884..638b6e96b5025b 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -13,7 +13,7 @@ import warnings from test import support from test.support import (script_helper, requires_debug_ranges, - requires_specialization, Py_C_RECURSION_LIMIT) + requires_specialization, get_c_recursion_limit) from test.support.bytecode_helper import instructions_with_positions from test.support.os_helper import FakePath @@ -114,7 +114,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = int(Py_C_RECURSION_LIMIT * 0.9) + repeat = int(get_c_recursion_limit() * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -634,9 +634,10 @@ def test_yet_more_evil_still_undecodable(self): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): # Expected limit is Py_C_RECURSION_LIMIT - fail_depth = Py_C_RECURSION_LIMIT + 1 - crash_depth = Py_C_RECURSION_LIMIT * 100 - success_depth = int(Py_C_RECURSION_LIMIT * 0.8) + limit = get_c_recursion_limit() + fail_depth = limit + 1 + crash_depth = limit * 100 + success_depth = int(limit * 0.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 620d0ca4f4c2da..e5dba7cdc570a8 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper, Py_C_RECURSION_LIMIT +from test.support import import_helper, get_c_recursion_limit class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index cad568b6ac4c2d..d9881611c19c43 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -2,7 +2,7 @@ import copy import pickle import unittest -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class DictSetTest(unittest.TestCase): @@ -279,7 +279,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(Py_C_RECURSION_LIMIT//2 + 100): + for i in range(get_c_recursion_limit()//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 20122679223843..b4fc290b1f32b6 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): e = ExceptionGroup('eg', [e]) return e diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 6ad6acc61563e5..36fd89dbb8896c 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1424,7 +1424,7 @@ def gen(): next(generator) recursionlimit = sys.getrecursionlimit() try: - recurse(support.EXCEEDS_RECURSION_LIMIT) + recurse(support.exceeds_recursion_limit()) finally: sys.setrecursionlimit(recursionlimit) print('Done.') diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 3ba4929dd1b133..c48c399a10c853 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1867,7 +1867,7 @@ def fib(n): return fib(n-1) + fib(n-2) if not support.Py_DEBUG: - depth = support.Py_C_RECURSION_LIMIT*2//7 + depth = support.get_c_recursion_limit()*2//7 with support.infinite_recursion(): fib(depth) if self.module == c_functools: diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 7f759fb3317146..95a119ba683e09 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - for cnt in range(support.EXCEEDS_RECURSION_LIMIT): + for cnt in range(support.exceeds_recursion_limit()): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 615568e6af2102..64ee1ba867d592 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -118,7 +118,7 @@ def test_code(self): def test_many_codeobjects(self): # Issue2957: bad recursion count on code objects # more than MAX_MARSHAL_STACK_DEPTH - count = support.EXCEEDS_RECURSION_LIMIT + count = support.exceeds_recursion_limit() codes = (ExceptionTestCase.test_exceptions.__code__,) * count marshal.loads(marshal.dumps(codes)) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 125f40227118f6..ded1d9224d82d9 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -3039,7 +3039,7 @@ def test_trace_unpack_long_sequence(self): def test_trace_lots_of_globals(self): - count = min(1000, int(support.Py_C_RECURSION_LIMIT * 0.8)) + count = min(1000, int(support.get_c_recursion_limit() * 0.8)) code = """if 1: def f(): From 59864edd572b5c0cc3be58087a9ea3a700226146 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 8 Apr 2024 10:46:56 -0400 Subject: [PATCH 110/143] gh-117552: Add timeout in HTTPHandlerTest (#117553) --- Lib/test/test_logging.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 7b5bc6b6a74180..c3c4de06fa0f09 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -2191,7 +2191,8 @@ def test_output(self): self.handled.clear() msg = "sp\xe4m" logger.error(msg) - self.handled.wait() + handled = self.handled.wait(support.SHORT_TIMEOUT) + self.assertTrue(handled, "HTTP request timed out") self.assertEqual(self.log_data.path, '/frob') self.assertEqual(self.command, method) if method == 'GET': From 26a680a58524fe39eecb243e37adfa6e157466f6 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 8 Apr 2024 10:47:42 -0400 Subject: [PATCH 111/143] gh-117293: Fix race condition in run_workers.py (#117298) The worker thread may still be alive after it enqueues it's last result, which can lead to a delay of 30 seconds after the test finishes. This happens much more frequently in the free-threaded build with the GIL disabled. This changes run_workers.py to track of live workers by enqueueing a `WorkerExited()` instance before the worker exits. --- Lib/test/libregrtest/run_workers.py | 37 ++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 9cfe1b9d6fd07d..235047cf2e563c 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -79,8 +79,12 @@ class MultiprocessResult: err_msg: str | None = None +class WorkerThreadExited: + """Indicates that a worker thread has exited""" + ExcStr = str QueueOutput = tuple[Literal[False], MultiprocessResult] | tuple[Literal[True], ExcStr] +QueueContent = QueueOutput | WorkerThreadExited class ExitThread(Exception): @@ -376,8 +380,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: def run(self) -> None: fail_fast = self.runtests.fail_fast fail_env_changed = self.runtests.fail_env_changed - while not self._stopped: - try: + try: + while not self._stopped: try: test_name = next(self.pending) except StopIteration: @@ -396,11 +400,12 @@ def run(self) -> None: if mp_result.result.must_stop(fail_fast, fail_env_changed): break - except ExitThread: - break - except BaseException: - self.output.put((True, traceback.format_exc())) - break + except ExitThread: + pass + except BaseException: + self.output.put((True, traceback.format_exc())) + finally: + self.output.put(WorkerThreadExited()) def _wait_completed(self) -> None: popen = self._popen @@ -458,8 +463,9 @@ def __init__(self, num_workers: int, runtests: RunTests, self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results + self.live_worker_count = 0 - self.output: queue.Queue[QueueOutput] = queue.Queue() + self.output: queue.Queue[QueueContent] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) self.timeout = runtests.timeout @@ -497,6 +503,7 @@ def start_workers(self) -> None: self.log(msg) for worker in self.workers: worker.start() + self.live_worker_count += 1 def stop_workers(self) -> None: start_time = time.monotonic() @@ -511,14 +518,18 @@ def _get_result(self) -> QueueOutput | None: # bpo-46205: check the status of workers every iteration to avoid # waiting forever on an empty queue. - while any(worker.is_alive() for worker in self.workers): + while self.live_worker_count > 0: if use_faulthandler: faulthandler.dump_traceback_later(MAIN_PROCESS_TIMEOUT, exit=True) # wait for a thread try: - return self.output.get(timeout=PROGRESS_UPDATE) + result = self.output.get(timeout=PROGRESS_UPDATE) + if isinstance(result, WorkerThreadExited): + self.live_worker_count -= 1 + continue + return result except queue.Empty: pass @@ -528,12 +539,6 @@ def _get_result(self) -> QueueOutput | None: if running: self.log(running) - # all worker threads are done: consume pending results - try: - return self.output.get(timeout=0) - except queue.Empty: - return None - def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result pgo = self.runtests.pgo From e16062dd3428a5846344e0a8c6ee2f352d34ce1b Mon Sep 17 00:00:00 2001 From: Laurie O Date: Tue, 9 Apr 2024 00:50:54 +1000 Subject: [PATCH 112/143] gh-96471: Correct documentation for asyncio queue shutdown (#117621) --- Doc/library/asyncio-queue.rst | 7 ++++--- Doc/whatsnew/3.13.rst | 2 +- Lib/asyncio/queues.py | 6 ++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index 030d4310942d7a..9b579cc1d5fdfe 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -106,9 +106,10 @@ Queue raise once the queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise immediately instead. - All blocked callers of :meth:`~Queue.put` will be unblocked. If - *immediate* is true, also unblock callers of :meth:`~Queue.get` - and :meth:`~Queue.join`. + All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` + will be unblocked. If *immediate* is true, a task will be marked + as done for each remaining item in the queue, which may unblock + callers of :meth:`~Queue.join`. .. versionadded:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 9e40bf04c49bfa..0fe2dafbfd6f02 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -298,7 +298,7 @@ asyncio * Add :meth:`asyncio.Queue.shutdown` (along with :exc:`asyncio.QueueShutDown`) for queue termination. - (Contributed by Laurie Opperman in :gh:`104228`.) + (Contributed by Laurie Opperman and Yves Duprat in :gh:`104228`.) base64 ------ diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index b8156704b8fc23..2f3865114a84f9 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -256,8 +256,9 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() will be unblocked, and also get() - and join() if 'immediate'. + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). """ self._is_shutdown = True if immediate: @@ -267,6 +268,7 @@ def shutdown(self, immediate=False): self._unfinished_tasks -= 1 if self._unfinished_tasks == 0: self._finished.set() + # All getters need to re-check queue-empty to raise ShutDown while self._getters: getter = self._getters.popleft() if not getter.done(): From df7317904849a41d51db39d92c5d431a18e22637 Mon Sep 17 00:00:00 2001 From: mpage Date: Mon, 8 Apr 2024 07:58:38 -0700 Subject: [PATCH 113/143] gh-111926: Make weakrefs thread-safe in free-threaded builds (#117168) Most mutable data is protected by a striped lock that is keyed on the referenced object's address. The weakref's hash is protected using the weakref's per-object lock. Note that this only affects free-threaded builds. Apart from some minor refactoring, the added code is all either gated by `ifdef`s or is a no-op (e.g. `Py_BEGIN_CRITICAL_SECTION`). --- Include/cpython/weakrefobject.h | 8 + Include/internal/pycore_interp.h | 7 + Include/internal/pycore_object.h | 40 +- .../internal/pycore_pyatomic_ft_wrappers.h | 5 + Include/internal/pycore_weakref.h | 73 ++- Lib/test/test_sys.py | 8 +- Lib/test/test_weakref.py | 19 + Modules/_sqlite/blob.c | 5 +- Modules/_sqlite/connection.c | 4 +- Modules/_ssl.c | 13 +- Modules/_ssl/debughelpers.c | 6 +- Modules/_weakref.c | 42 +- Modules/clinic/_weakref.c.h | 20 +- Objects/dictobject.c | 8 +- Objects/typeobject.c | 12 +- Objects/weakrefobject.c | 539 ++++++++++-------- Python/pystate.c | 9 + 17 files changed, 491 insertions(+), 327 deletions(-) diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 1559e2def61260..9a796098c6b48f 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -30,6 +30,14 @@ struct _PyWeakReference { PyWeakReference *wr_prev; PyWeakReference *wr_next; vectorcallfunc vectorcall; + +#ifdef Py_GIL_DISABLED + /* Pointer to the lock used when clearing in free-threaded builds. + * Normally this can be derived from wr_object, but in some cases we need + * to lock after wr_object has been set to Py_None. + */ + struct _PyMutex *weakrefs_lock; +#endif }; Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index b5cea863ff35dc..1bb123b8607edd 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -59,6 +59,12 @@ struct _stoptheworld_state { PyThreadState *requester; // Thread that requested the pause (may be NULL). }; +#ifdef Py_GIL_DISABLED +// This should be prime but otherwise the choice is arbitrary. A larger value +// increases concurrency at the expense of memory. +# define NUM_WEAKREF_LIST_LOCKS 127 +#endif + /* cross-interpreter data registry */ /* Tracks some rare events per-interpreter, used by the optimizer to turn on/off @@ -203,6 +209,7 @@ struct _is { #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; struct _brc_state brc; // biased reference counting state + PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; #endif // Per-interpreter state for the obmalloc allocator. For the main diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 4fc5e9bf653c1c..1e1b664000f108 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -426,7 +426,7 @@ _Py_TryIncRefShared(PyObject *op) /* Tries to incref the object op and ensures that *src still points to it. */ static inline int -_Py_TryIncref(PyObject **src, PyObject *op) +_Py_TryIncrefCompare(PyObject **src, PyObject *op) { if (_Py_TryIncrefFast(op)) { return 1; @@ -452,7 +452,7 @@ _Py_XGetRef(PyObject **ptr) if (value == NULL) { return value; } - if (_Py_TryIncref(ptr, value)) { + if (_Py_TryIncrefCompare(ptr, value)) { return value; } } @@ -467,7 +467,7 @@ _Py_TryXGetRef(PyObject **ptr) if (value == NULL) { return value; } - if (_Py_TryIncref(ptr, value)) { + if (_Py_TryIncrefCompare(ptr, value)) { return value; } return NULL; @@ -506,8 +506,42 @@ _Py_XNewRefWithLock(PyObject *obj) return _Py_NewRefWithLock(obj); } +static inline void +_PyObject_SetMaybeWeakref(PyObject *op) +{ + if (_Py_IsImmortal(op)) { + return; + } + for (;;) { + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) { + // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states. + return; + } + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) { + return; + } + } +} + #endif +/* Tries to incref op and returns 1 if successful or 0 otherwise. */ +static inline int +_Py_TryIncref(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return _Py_TryIncrefFast(op) || _Py_TryIncRefShared(op); +#else + if (Py_REFCNT(op) > 0) { + Py_INCREF(op); + return 1; + } + return 0; +#endif +} + #ifdef Py_REF_DEBUG extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *); extern void _Py_FinalizeRefTotal(_PyRuntimeState *); diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index e441600d54e1aa..2514f51f1b0086 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -20,9 +20,12 @@ extern "C" { #endif #ifdef Py_GIL_DISABLED +#define FT_ATOMIC_LOAD_PTR(value) _Py_atomic_load_ptr(&value) #define FT_ATOMIC_LOAD_SSIZE(value) _Py_atomic_load_ssize(&value) #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) \ _Py_atomic_load_ssize_relaxed(&value) +#define FT_ATOMIC_STORE_PTR(value, new_value) \ + _Py_atomic_store_ptr(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ @@ -30,8 +33,10 @@ extern "C" { #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) #else +#define FT_ATOMIC_LOAD_PTR(value) value #define FT_ATOMIC_LOAD_SSIZE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value +#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index dea267b49039e7..e057a27340f718 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -9,7 +9,35 @@ extern "C" { #endif #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_lock.h" #include "pycore_object.h" // _Py_REF_IS_MERGED() +#include "pycore_pyatomic_ft_wrappers.h" + +#ifdef Py_GIL_DISABLED + +#define WEAKREF_LIST_LOCK(obj) \ + _PyInterpreterState_GET() \ + ->weakref_locks[((uintptr_t)obj) % NUM_WEAKREF_LIST_LOCKS] + +// Lock using the referenced object +#define LOCK_WEAKREFS(obj) \ + PyMutex_LockFlags(&WEAKREF_LIST_LOCK(obj), _Py_LOCK_DONT_DETACH) +#define UNLOCK_WEAKREFS(obj) PyMutex_Unlock(&WEAKREF_LIST_LOCK(obj)) + +// Lock using a weakref +#define LOCK_WEAKREFS_FOR_WR(wr) \ + PyMutex_LockFlags(wr->weakrefs_lock, _Py_LOCK_DONT_DETACH) +#define UNLOCK_WEAKREFS_FOR_WR(wr) PyMutex_Unlock(wr->weakrefs_lock) + +#else + +#define LOCK_WEAKREFS(obj) +#define UNLOCK_WEAKREFS(obj) + +#define LOCK_WEAKREFS_FOR_WR(wr) +#define UNLOCK_WEAKREFS_FOR_WR(wr) + +#endif static inline int _is_dead(PyObject *obj) { @@ -30,53 +58,64 @@ static inline int _is_dead(PyObject *obj) static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) { assert(PyWeakref_Check(ref_obj)); - PyObject *ret = NULL; - Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); - PyObject *obj = ref->wr_object; + PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); if (obj == Py_None) { // clear_weakref() was called - goto end; + return NULL; } - if (_is_dead(obj)) { - goto end; + LOCK_WEAKREFS(obj); +#ifdef Py_GIL_DISABLED + if (ref->wr_object == Py_None) { + // clear_weakref() was called + UNLOCK_WEAKREFS(obj); + return NULL; } -#if !defined(Py_GIL_DISABLED) - assert(Py_REFCNT(obj) > 0); #endif - ret = Py_NewRef(obj); -end: - Py_END_CRITICAL_SECTION(); - return ret; + if (_Py_TryIncref(obj)) { + UNLOCK_WEAKREFS(obj); + return obj; + } + UNLOCK_WEAKREFS(obj); + return NULL; } static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) { assert(PyWeakref_Check(ref_obj)); int ret = 0; - Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); - PyObject *obj = ref->wr_object; + PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); if (obj == Py_None) { // clear_weakref() was called ret = 1; } else { + LOCK_WEAKREFS(obj); // See _PyWeakref_GET_REF() for the rationale of this test +#ifdef Py_GIL_DISABLED + ret = (ref->wr_object == Py_None) || _is_dead(obj); +#else ret = _is_dead(obj); +#endif + UNLOCK_WEAKREFS(obj); } - Py_END_CRITICAL_SECTION(); return ret; } -extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyWeakReference *head); +extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyObject *obj); + +// Clear all the weak references to obj but leave their callbacks uncalled and +// intact. +extern void _PyWeakref_ClearWeakRefsExceptCallbacks(PyObject *obj); extern void _PyWeakref_ClearRef(PyWeakReference *self); +PyAPI_FUNC(int) _PyWeakref_IsDead(PyObject *weakref); + #ifdef __cplusplus } #endif #endif /* !Py_INTERNAL_WEAKREF_H */ - diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 6a66df4e897e3f..ab26bf56d9ced9 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1708,11 +1708,15 @@ class newstyleclass(object): pass # TODO: add check that forces layout of unicodefields # weakref import weakref - check(weakref.ref(int), size('2Pn3P')) + if support.Py_GIL_DISABLED: + expected = size('2Pn4P') + else: + expected = size('2Pn3P') + check(weakref.ref(int), expected) # weakproxy # XXX # weakcallableproxy - check(weakref.proxy(int), size('2Pn3P')) + check(weakref.proxy(int), expected) def check_slots(self, obj, base, extra): expected = sys.getsizeof(base) + struct.calcsize(extra) diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 6fbd292c1e6793..d0e8df4ea82802 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1907,6 +1907,25 @@ def test_threaded_weak_valued_consistency(self): self.assertEqual(len(d), 1) o = None # lose ref + @support.cpython_only + def test_weak_valued_consistency(self): + # A single-threaded, deterministic repro for issue #28427: old keys + # should not remove new values from WeakValueDictionary. This relies on + # an implementation detail of CPython's WeakValueDictionary (its + # underlying dictionary of KeyedRefs) to reproduce the issue. + d = weakref.WeakValueDictionary() + with support.disable_gc(): + d[10] = RefCycle() + # Keep the KeyedRef alive after it's replaced so that GC will invoke + # the callback. + wr = d.data[10] + # Replace the value with something that isn't cyclic garbage + o = RefCycle() + d[10] = o + # Trigger GC, which will invoke the callback for `wr` + gc.collect() + self.assertEqual(len(d), 1) + def check_threaded_weak_dict_copy(self, type_, deepcopy): # `type_` should be either WeakKeyDictionary or WeakValueDictionary. # `deepcopy` should be either True or False. diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f099020c5f4e6f..7deb58bf1b9b82 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -4,7 +4,6 @@ #include "blob.h" #include "util.h" -#include "pycore_weakref.h" // _PyWeakref_GET_REF() #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/blob.c.h" @@ -102,8 +101,8 @@ pysqlite_close_all_blobs(pysqlite_Connection *self) { for (int i = 0; i < PyList_GET_SIZE(self->blobs); i++) { PyObject *weakref = PyList_GET_ITEM(self->blobs, i); - PyObject *blob = _PyWeakref_GET_REF(weakref); - if (blob == NULL) { + PyObject *blob; + if (!PyWeakref_GetRef(weakref, &blob)) { continue; } close_blob((pysqlite_Blob *)blob); diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index f97afcf5fcf16e..74984ca5365743 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -38,7 +38,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() -#include "pycore_weakref.h" // _PyWeakref_IS_DEAD() +#include "pycore_weakref.h" #include @@ -1065,7 +1065,7 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) for (Py_ssize_t i = 0; i < PyList_Size(self->cursors); i++) { PyObject* weakref = PyList_GetItem(self->cursors, i); - if (_PyWeakref_IS_DEAD(weakref)) { + if (_PyWeakref_IsDead(weakref)) { continue; } if (PyList_Append(new_list, weakref) != 0) { diff --git a/Modules/_ssl.c b/Modules/_ssl.c index fbf914c4321922..f7fdbf4b6f90cb 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -29,7 +29,6 @@ #include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_time.h" // _PyDeadline_Init() -#include "pycore_weakref.h" // _PyWeakref_GET_REF() /* Include symbols from _socket module */ #include "socketmodule.h" @@ -392,8 +391,8 @@ typedef enum { // Return a borrowed reference. static inline PySocketSockObject* GET_SOCKET(PySSLSocket *obj) { if (obj->Socket) { - PyObject *sock = _PyWeakref_GET_REF(obj->Socket); - if (sock != NULL) { + PyObject *sock; + if (PyWeakref_GetRef(obj->Socket, &sock)) { // GET_SOCKET() returns a borrowed reference Py_DECREF(sock); } @@ -2205,8 +2204,8 @@ PySSL_get_owner(PySSLSocket *self, void *c) if (self->owner == NULL) { Py_RETURN_NONE; } - PyObject *owner = _PyWeakref_GET_REF(self->owner); - if (owner == NULL) { + PyObject *owner; + if (!PyWeakref_GetRef(self->owner, &owner)) { Py_RETURN_NONE; } return owner; @@ -4433,9 +4432,9 @@ _servername_callback(SSL *s, int *al, void *args) * will be passed. If both do not exist only then the C-level object is * passed. */ if (ssl->owner) - ssl_socket = _PyWeakref_GET_REF(ssl->owner); + PyWeakref_GetRef(ssl->owner, &ssl_socket); else if (ssl->Socket) - ssl_socket = _PyWeakref_GET_REF(ssl->Socket); + PyWeakref_GetRef(ssl->Socket, &ssl_socket); else ssl_socket = Py_NewRef(ssl); diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 07e9ce7a6fce2d..9c87f8b4d21e68 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -28,12 +28,12 @@ _PySSL_msg_callback(int write_p, int version, int content_type, PyObject *ssl_socket; /* ssl.SSLSocket or ssl.SSLObject */ if (ssl_obj->owner) - ssl_socket = _PyWeakref_GET_REF(ssl_obj->owner); + PyWeakref_GetRef(ssl_obj->owner, &ssl_socket); else if (ssl_obj->Socket) - ssl_socket = _PyWeakref_GET_REF(ssl_obj->Socket); + PyWeakref_GetRef(ssl_obj->Socket, &ssl_socket); else ssl_socket = (PyObject *)Py_NewRef(ssl_obj); - assert(ssl_socket != NULL); // _PyWeakref_GET_REF() can return NULL + assert(ssl_socket != NULL); // PyWeakref_GetRef() can return NULL /* assume that OpenSSL verifies all payload and buf len is of sufficient length */ diff --git a/Modules/_weakref.c b/Modules/_weakref.c index 7225dbc9ce4a1b..1ea3ed5e40b761 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -14,7 +14,6 @@ module _weakref #include "clinic/_weakref.c.h" /*[clinic input] -@critical_section object _weakref.getweakrefcount -> Py_ssize_t object: object @@ -25,14 +24,9 @@ Return the number of weak references to 'object'. static Py_ssize_t _weakref_getweakrefcount_impl(PyObject *module, PyObject *object) -/*[clinic end generated code: output=301806d59558ff3e input=6535a580f1d0ebdc]*/ +/*[clinic end generated code: output=301806d59558ff3e input=7d4d04fcaccf64d5]*/ { - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) { - return 0; - } - PyWeakReference **list = GET_WEAKREFS_LISTPTR(object); - Py_ssize_t count = _PyWeakref_GetWeakrefCount(*list); - return count; + return _PyWeakref_GetWeakrefCount(object); } @@ -77,7 +71,6 @@ _weakref__remove_dead_weakref_impl(PyObject *module, PyObject *dct, /*[clinic input] -@critical_section object _weakref.getweakrefs object: object / @@ -86,26 +79,39 @@ Return a list of all weak reference objects pointing to 'object'. [clinic start generated code]*/ static PyObject * -_weakref_getweakrefs_impl(PyObject *module, PyObject *object) -/*[clinic end generated code: output=5ec268989fb8f035 input=3dea95b8f5b31bbb]*/ +_weakref_getweakrefs(PyObject *module, PyObject *object) +/*[clinic end generated code: output=25c7731d8e011824 input=00c6d0e5d3206693]*/ { if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) { return PyList_New(0); } - PyWeakReference **list = GET_WEAKREFS_LISTPTR(object); - Py_ssize_t count = _PyWeakref_GetWeakrefCount(*list); - - PyObject *result = PyList_New(count); + PyObject *result = PyList_New(0); if (result == NULL) { return NULL; } - PyWeakReference *current = *list; - for (Py_ssize_t i = 0; i < count; ++i) { - PyList_SET_ITEM(result, i, Py_NewRef(current)); + LOCK_WEAKREFS(object); + PyWeakReference *current = *GET_WEAKREFS_LISTPTR(object); + while (current != NULL) { + PyObject *curobj = (PyObject *) current; + if (_Py_TryIncref(curobj)) { + if (PyList_Append(result, curobj)) { + UNLOCK_WEAKREFS(object); + Py_DECREF(curobj); + Py_DECREF(result); + return NULL; + } + else { + // Undo our _Py_TryIncref. This is safe to do with the lock + // held in free-threaded builds; the list holds a reference to + // curobj so we're guaranteed not to invoke the destructor. + Py_DECREF(curobj); + } + } current = current->wr_next; } + UNLOCK_WEAKREFS(object); return result; } diff --git a/Modules/clinic/_weakref.c.h b/Modules/clinic/_weakref.c.h index 550b6c4d71a015..8d7bc5dc936610 100644 --- a/Modules/clinic/_weakref.c.h +++ b/Modules/clinic/_weakref.c.h @@ -2,7 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_weakref_getweakrefcount__doc__, @@ -23,9 +22,7 @@ _weakref_getweakrefcount(PyObject *module, PyObject *object) PyObject *return_value = NULL; Py_ssize_t _return_value; - Py_BEGIN_CRITICAL_SECTION(object); _return_value = _weakref_getweakrefcount_impl(module, object); - Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -79,21 +76,6 @@ PyDoc_STRVAR(_weakref_getweakrefs__doc__, #define _WEAKREF_GETWEAKREFS_METHODDEF \ {"getweakrefs", (PyCFunction)_weakref_getweakrefs, METH_O, _weakref_getweakrefs__doc__}, -static PyObject * -_weakref_getweakrefs_impl(PyObject *module, PyObject *object); - -static PyObject * -_weakref_getweakrefs(PyObject *module, PyObject *object) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(object); - return_value = _weakref_getweakrefs_impl(module, object); - Py_END_CRITICAL_SECTION(); - - return return_value; -} - PyDoc_STRVAR(_weakref_proxy__doc__, "proxy($module, object, callback=None, /)\n" "--\n" @@ -130,4 +112,4 @@ _weakref_proxy(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=d5d30707212a9870 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=60f59adc1dc9eab8 input=a9049054013a1b77]*/ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b62d39ad6c5192..9218b1aa470663 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1286,7 +1286,7 @@ Py_ssize_t compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject assert(!PyUnicode_CheckExact(key)); if (startkey != NULL) { - if (!_Py_TryIncref(&ep->me_key, startkey)) { + if (!_Py_TryIncrefCompare(&ep->me_key, startkey)) { return DKIX_KEY_CHANGED; } @@ -1334,7 +1334,7 @@ compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, return unicode_get_hash(startkey) == hash && unicode_eq(startkey, key); } else { - if (!_Py_TryIncref(&ep->me_key, startkey)) { + if (!_Py_TryIncrefCompare(&ep->me_key, startkey)) { return DKIX_KEY_CHANGED; } if (unicode_get_hash(startkey) == hash && unicode_eq(startkey, key)) { @@ -1364,7 +1364,7 @@ Py_ssize_t compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, } Py_ssize_t ep_hash = _Py_atomic_load_ssize_relaxed(&ep->me_hash); if (ep_hash == hash) { - if (startkey == NULL || !_Py_TryIncref(&ep->me_key, startkey)) { + if (startkey == NULL || !_Py_TryIncrefCompare(&ep->me_key, startkey)) { return DKIX_KEY_CHANGED; } int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); @@ -5308,7 +5308,7 @@ acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, } if (out_value) { - if (!_Py_TryIncref(value_loc, value)) { + if (!_Py_TryIncrefCompare(value_loc, value)) { if (out_key) { Py_DECREF(*out_key); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 51ceb7d7de1cb6..e9f2d2577e9fab 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -378,7 +378,7 @@ _PyType_GetMRO(PyTypeObject *self) if (mro == NULL) { return NULL; } - if (_Py_TryIncref(&self->tp_mro, mro)) { + if (_Py_TryIncrefCompare(&self->tp_mro, mro)) { return mro; } @@ -2193,15 +2193,7 @@ subtype_dealloc(PyObject *self) finalizers since they might rely on part of the object being finalized that has already been destroyed. */ if (type->tp_weaklistoffset && !base->tp_weaklistoffset) { - /* Modeled after GET_WEAKREFS_LISTPTR(). - - This is never triggered for static types so we can avoid the - (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). */ - PyWeakReference **list = \ - _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(self); - while (*list) { - _PyWeakref_ClearRef(*list); - } + _PyWeakref_ClearWeakRefsExceptCallbacks(self); } } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index d8dd6aea3aff02..206107e8505dc7 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1,24 +1,58 @@ #include "Python.h" +#include "pycore_critical_section.h" +#include "pycore_lock.h" #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GET_WEAKREFS_LISTPTR() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pystate.h" #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#ifdef Py_GIL_DISABLED +/* + * Thread-safety for free-threaded builds + * ====================================== + * + * In free-threaded builds we need to protect mutable state of: + * + * - The weakref (wr_object, hash, wr_callback) + * - The referenced object (its head-of-list pointer) + * - The linked list of weakrefs + * + * For now we've chosen to address this in a straightforward way: + * + * - The weakref's hash is protected using the weakref's per-object lock. + * - The other mutable is protected by a striped lock keyed on the referenced + * object's address. + * - The striped lock must be locked using `_Py_LOCK_DONT_DETACH` in order to + * support atomic deletion from WeakValueDictionaries. As a result, we must + * be careful not to perform any operations that could suspend while the + * lock is held. + * + * Since the world is stopped when the GC runs, it is free to clear weakrefs + * without acquiring any locks. + */ +#endif #define GET_WEAKREFS_LISTPTR(o) \ ((PyWeakReference **) _PyObject_GET_WEAKREFS_LISTPTR(o)) Py_ssize_t -_PyWeakref_GetWeakrefCount(PyWeakReference *head) +_PyWeakref_GetWeakrefCount(PyObject *obj) { - Py_ssize_t count = 0; + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + return 0; + } + LOCK_WEAKREFS(obj); + Py_ssize_t count = 0; + PyWeakReference *head = *GET_WEAKREFS_LISTPTR(obj); while (head != NULL) { ++count; head = head->wr_next; } + UNLOCK_WEAKREFS(obj); return count; } @@ -33,54 +67,55 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) self->wr_next = NULL; self->wr_callback = Py_XNewRef(callback); self->vectorcall = weakref_vectorcall; +#ifdef Py_GIL_DISABLED + self->weakrefs_lock = &WEAKREF_LIST_LOCK(ob); + _PyObject_SetMaybeWeakref(ob); + _PyObject_SetMaybeWeakref((PyObject *)self); +#endif } -static PyWeakReference * -new_weakref(PyObject *ob, PyObject *callback) -{ - PyWeakReference *result; - - result = PyObject_GC_New(PyWeakReference, &_PyWeakref_RefType); - if (result) { - init_weakref(result, ob, callback); - PyObject_GC_Track(result); - } - return result; -} - - -/* This function clears the passed-in reference and removes it from the - * list of weak references for the referent. This is the only code that - * removes an item from the doubly-linked list of weak references for an - * object; it is also responsible for clearing the callback slot. - */ +// Clear the weakref and steal its callback into `callback`, if provided. static void -clear_weakref(PyWeakReference *self) +clear_weakref_lock_held(PyWeakReference *self, PyObject **callback) { - PyObject *callback = self->wr_callback; - if (self->wr_object != Py_None) { PyWeakReference **list = GET_WEAKREFS_LISTPTR(self->wr_object); - - if (*list == self) - /* If 'self' is the end of the list (and thus self->wr_next == NULL) - then the weakref list itself (and thus the value of *list) will - end up being set to NULL. */ - *list = self->wr_next; - self->wr_object = Py_None; - if (self->wr_prev != NULL) + if (*list == self) { + /* If 'self' is the end of the list (and thus self->wr_next == + NULL) then the weakref list itself (and thus the value of *list) + will end up being set to NULL. */ + FT_ATOMIC_STORE_PTR(*list, self->wr_next); + } + FT_ATOMIC_STORE_PTR(self->wr_object, Py_None); + if (self->wr_prev != NULL) { self->wr_prev->wr_next = self->wr_next; - if (self->wr_next != NULL) + } + if (self->wr_next != NULL) { self->wr_next->wr_prev = self->wr_prev; + } self->wr_prev = NULL; self->wr_next = NULL; } if (callback != NULL) { - Py_DECREF(callback); + *callback = self->wr_callback; self->wr_callback = NULL; } } +// Clear the weakref and its callback +static void +clear_weakref(PyWeakReference *self) +{ + PyObject *callback = NULL; + // self->wr_object may be Py_None if the GC cleared the weakref, so lock + // using the pointer in the weakref. + LOCK_WEAKREFS_FOR_WR(self); + clear_weakref_lock_held(self, &callback); + UNLOCK_WEAKREFS_FOR_WR(self); + Py_XDECREF(callback); +} + + /* Cyclic gc uses this to *just* clear the passed-in reference, leaving * the callback intact and uncalled. It must be possible to call self's * tp_dealloc() after calling this, so self has to be left in a sane enough @@ -95,15 +130,9 @@ clear_weakref(PyWeakReference *self) void _PyWeakref_ClearRef(PyWeakReference *self) { - PyObject *callback; - assert(self != NULL); assert(PyWeakref_Check(self)); - /* Preserve and restore the callback around clear_weakref. */ - callback = self->wr_callback; - self->wr_callback = NULL; - clear_weakref(self); - self->wr_callback = callback; + clear_weakref_lock_held(self, NULL); } static void @@ -126,7 +155,11 @@ gc_traverse(PyWeakReference *self, visitproc visit, void *arg) static int gc_clear(PyWeakReference *self) { - clear_weakref(self); + PyObject *callback; + // The world is stopped during GC in free-threaded builds. It's safe to + // call this without holding the lock. + clear_weakref_lock_held(self, &callback); + Py_XDECREF(callback); return 0; } @@ -150,7 +183,7 @@ weakref_vectorcall(PyObject *self, PyObject *const *args, } static Py_hash_t -weakref_hash(PyWeakReference *self) +weakref_hash_lock_held(PyWeakReference *self) { if (self->hash != -1) return self->hash; @@ -164,6 +197,15 @@ weakref_hash(PyWeakReference *self) return self->hash; } +static Py_hash_t +weakref_hash(PyWeakReference *self) +{ + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(self); + hash = weakref_hash_lock_held(self); + Py_END_CRITICAL_SECTION(); + return hash; +} static PyObject * weakref_repr(PyObject *self) @@ -276,6 +318,128 @@ insert_head(PyWeakReference *newref, PyWeakReference **list) *list = newref; } +/* See if we can reuse either the basic ref or proxy in list instead of + * creating a new weakref + */ +static PyWeakReference * +try_reuse_basic_ref(PyWeakReference *list, PyTypeObject *type, + PyObject *callback) +{ + if (callback != NULL) { + return NULL; + } + + PyWeakReference *ref, *proxy; + get_basic_refs(list, &ref, &proxy); + + PyWeakReference *cand = NULL; + if (type == &_PyWeakref_RefType) { + cand = ref; + } + if ((type == &_PyWeakref_ProxyType) || + (type == &_PyWeakref_CallableProxyType)) { + cand = proxy; + } + + if (cand != NULL && _Py_TryIncref((PyObject *) cand)) { + return cand; + } + return NULL; +} + +static int +is_basic_ref(PyWeakReference *ref) +{ + return (ref->wr_callback == NULL) && PyWeakref_CheckRefExact(ref); +} + +static int +is_basic_proxy(PyWeakReference *proxy) +{ + return (proxy->wr_callback == NULL) && PyWeakref_CheckProxy(proxy); +} + +static int +is_basic_ref_or_proxy(PyWeakReference *wr) +{ + return is_basic_ref(wr) || is_basic_proxy(wr); +} + +/* Insert `newref` in the appropriate position in `list` */ +static void +insert_weakref(PyWeakReference *newref, PyWeakReference **list) +{ + PyWeakReference *ref, *proxy; + get_basic_refs(*list, &ref, &proxy); + + PyWeakReference *prev; + if (is_basic_ref(newref)) { + prev = NULL; + } + else if (is_basic_proxy(newref)) { + prev = ref; + } + else { + prev = (proxy == NULL) ? ref : proxy; + } + + if (prev == NULL) { + insert_head(newref, list); + } + else { + insert_after(newref, prev); + } +} + +static PyWeakReference * +allocate_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) +{ + PyWeakReference *newref = (PyWeakReference *) type->tp_alloc(type, 0); + if (newref == NULL) { + return NULL; + } + init_weakref(newref, obj, callback); + return newref; +} + +static PyWeakReference * +get_or_create_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) +{ + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + PyErr_Format(PyExc_TypeError, + "cannot create weak reference to '%s' object", + Py_TYPE(obj)->tp_name); + return NULL; + } + if (callback == Py_None) + callback = NULL; + + PyWeakReference **list = GET_WEAKREFS_LISTPTR(obj); + if ((type == &_PyWeakref_RefType) || + (type == &_PyWeakref_ProxyType) || + (type == &_PyWeakref_CallableProxyType)) + { + LOCK_WEAKREFS(obj); + PyWeakReference *basic_ref = try_reuse_basic_ref(*list, type, callback); + if (basic_ref != NULL) { + UNLOCK_WEAKREFS(obj); + return basic_ref; + } + PyWeakReference *newref = allocate_weakref(type, obj, callback); + insert_weakref(newref, list); + UNLOCK_WEAKREFS(obj); + return newref; + } + else { + // We may not be able to safely allocate inside the lock + PyWeakReference *newref = allocate_weakref(type, obj, callback); + LOCK_WEAKREFS(obj); + insert_weakref(newref, list); + UNLOCK_WEAKREFS(obj); + return newref; + } +} + static int parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, PyObject **obp, PyObject **callbackp) @@ -286,54 +450,11 @@ parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, static PyObject * weakref___new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - PyWeakReference *self = NULL; PyObject *ob, *callback = NULL; - if (parse_weakref_init_args("__new__", args, kwargs, &ob, &callback)) { - PyWeakReference *ref, *proxy; - PyWeakReference **list; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - if (callback == Py_None) - callback = NULL; - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL && type == &_PyWeakref_RefType) { - if (ref != NULL) { - /* We can re-use an existing reference. */ - return Py_NewRef(ref); - } - } - /* We have to create a new reference. */ - /* Note: the tp_alloc() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - self = (PyWeakReference *) (type->tp_alloc(type, 0)); - if (self != NULL) { - init_weakref(self, ob, callback); - if (callback == NULL && type == &_PyWeakref_RefType) { - insert_head(self, list); - } - else { - PyWeakReference *prev; - - get_basic_refs(*list, &ref, &proxy); - prev = (proxy == NULL) ? ref : proxy; - if (prev == NULL) - insert_head(self, list); - else - insert_after(self, prev); - } - } + return (PyObject *)get_or_create_weakref(type, ob, callback); } - return (PyObject *)self; + return NULL; } static int @@ -562,8 +683,6 @@ static void proxy_dealloc(PyWeakReference *self) { PyObject_GC_UnTrack(self); - if (self->wr_callback != NULL) - PyObject_GC_UnTrack((PyObject *)self); clear_weakref(self); PyObject_GC_Del(self); } @@ -784,104 +903,21 @@ _PyWeakref_CallableProxyType = { proxy_iternext, /* tp_iternext */ }; - - PyObject * PyWeakref_NewRef(PyObject *ob, PyObject *callback) { - PyWeakReference *result = NULL; - PyWeakReference **list; - PyWeakReference *ref, *proxy; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == Py_None) - callback = NULL; - if (callback == NULL) - /* return existing weak reference if it exists */ - result = ref; - if (result != NULL) - Py_INCREF(result); - else { - /* We do not need to recompute ref/proxy; new_weakref() cannot - trigger GC. - */ - result = new_weakref(ob, callback); - if (result != NULL) { - if (callback == NULL) { - assert(ref == NULL); - insert_head(result, list); - } - else { - PyWeakReference *prev; - - prev = (proxy == NULL) ? ref : proxy; - if (prev == NULL) - insert_head(result, list); - else - insert_after(result, prev); - } - } - } - return (PyObject *) result; + return (PyObject *)get_or_create_weakref(&_PyWeakref_RefType, ob, + callback); } - PyObject * PyWeakref_NewProxy(PyObject *ob, PyObject *callback) { - PyWeakReference *result = NULL; - PyWeakReference **list; - PyWeakReference *ref, *proxy; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == Py_None) - callback = NULL; - if (callback == NULL) - /* attempt to return an existing weak reference if it exists */ - result = proxy; - if (result != NULL) - Py_INCREF(result); - else { - /* We do not need to recompute ref/proxy; new_weakref cannot - trigger GC. - */ - result = new_weakref(ob, callback); - if (result != NULL) { - PyWeakReference *prev; - - if (PyCallable_Check(ob)) { - Py_SET_TYPE(result, &_PyWeakref_CallableProxyType); - } - else { - Py_SET_TYPE(result, &_PyWeakref_ProxyType); - } - if (callback == NULL) { - prev = ref; - } - else - prev = (proxy == NULL) ? ref : proxy; - - if (prev == NULL) - insert_head(result, list); - else - insert_after(result, prev); - } + PyTypeObject *type = &_PyWeakref_ProxyType; + if (PyCallable_Check(ob)) { + type = &_PyWeakref_CallableProxyType; } - return (PyObject *) result; + return (PyObject *)get_or_create_weakref(type, ob, callback); } @@ -950,68 +986,73 @@ PyObject_ClearWeakRefs(PyObject *object) PyErr_BadInternalCall(); return; } + list = GET_WEAKREFS_LISTPTR(object); - /* Remove the callback-less basic and proxy references */ - if (*list != NULL && (*list)->wr_callback == NULL) { - clear_weakref(*list); - if (*list != NULL && (*list)->wr_callback == NULL) - clear_weakref(*list); + if (FT_ATOMIC_LOAD_PTR(list) == NULL) { + // Fast path for the common case + return; } - if (*list != NULL) { - PyWeakReference *current = *list; - Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); - PyObject *exc = PyErr_GetRaisedException(); - - if (count == 1) { - PyObject *callback = current->wr_callback; - - current->wr_callback = NULL; - clear_weakref(current); - if (callback != NULL) { - if (Py_REFCNT((PyObject *)current) > 0) { - handle_callback(current, callback); - } - Py_DECREF(callback); - } + + /* Remove the callback-less basic and proxy references, which always appear + at the head of the list. + */ + for (int done = 0; !done;) { + LOCK_WEAKREFS(object); + if (*list != NULL && is_basic_ref_or_proxy(*list)) { + PyObject *callback; + clear_weakref_lock_held(*list, &callback); + assert(callback == NULL); } - else { - PyObject *tuple; - Py_ssize_t i = 0; - - tuple = PyTuple_New(count * 2); - if (tuple == NULL) { - _PyErr_ChainExceptions1(exc); - return; - } + done = (*list == NULL) || !is_basic_ref_or_proxy(*list); + UNLOCK_WEAKREFS(object); + } - for (i = 0; i < count; ++i) { - PyWeakReference *next = current->wr_next; - - if (Py_REFCNT((PyObject *)current) > 0) { - PyTuple_SET_ITEM(tuple, i * 2, Py_NewRef(current)); - PyTuple_SET_ITEM(tuple, i * 2 + 1, current->wr_callback); - } - else { - Py_DECREF(current->wr_callback); - } - current->wr_callback = NULL; - clear_weakref(current); - current = next; - } - for (i = 0; i < count; ++i) { - PyObject *callback = PyTuple_GET_ITEM(tuple, i * 2 + 1); - - /* The tuple may have slots left to NULL */ - if (callback != NULL) { - PyObject *item = PyTuple_GET_ITEM(tuple, i * 2); - handle_callback((PyWeakReference *)item, callback); - } + /* Deal with non-canonical (subtypes or refs with callbacks) references. */ + Py_ssize_t num_weakrefs = _PyWeakref_GetWeakrefCount(object); + if (num_weakrefs == 0) { + return; + } + + PyObject *exc = PyErr_GetRaisedException(); + PyObject *tuple = PyTuple_New(num_weakrefs * 2); + if (tuple == NULL) { + _PyErr_ChainExceptions1(exc); + return; + } + + Py_ssize_t num_items = 0; + for (int done = 0; !done;) { + PyObject *callback = NULL; + LOCK_WEAKREFS(object); + PyWeakReference *cur = *list; + if (cur != NULL) { + clear_weakref_lock_held(cur, &callback); + if (_Py_TryIncref((PyObject *) cur)) { + assert(num_items / 2 < num_weakrefs); + PyTuple_SET_ITEM(tuple, num_items, (PyObject *) cur); + PyTuple_SET_ITEM(tuple, num_items + 1, callback); + num_items += 2; + callback = NULL; } - Py_DECREF(tuple); } - assert(!PyErr_Occurred()); - PyErr_SetRaisedException(exc); + done = (*list == NULL); + UNLOCK_WEAKREFS(object); + + Py_XDECREF(callback); } + + for (Py_ssize_t i = 0; i < num_items; i += 2) { + PyObject *callback = PyTuple_GET_ITEM(tuple, i + 1); + if (callback != NULL) { + PyObject *weakref = PyTuple_GET_ITEM(tuple, i); + handle_callback((PyWeakReference *)weakref, callback); + } + } + + Py_DECREF(tuple); + + assert(!PyErr_Occurred()); + PyErr_SetRaisedException(exc); } /* This function is called by _PyStaticType_Dealloc() to clear weak references. @@ -1025,10 +1066,30 @@ _PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type) { static_builtin_state *state = _PyStaticType_GetState(interp, type); PyObject **list = _PyStaticType_GET_WEAKREFS_LISTPTR(state); - while (*list != NULL) { - /* Note that clear_weakref() pops the first ref off the type's - weaklist before clearing its wr_object and wr_callback. - That is how we're able to loop over the list. */ - clear_weakref((PyWeakReference *)*list); + // This is safe to do without holding the lock in free-threaded builds; + // there is only one thread running and no new threads can be created. + while (*list) { + _PyWeakref_ClearRef((PyWeakReference *)*list); + } +} + +void +_PyWeakref_ClearWeakRefsExceptCallbacks(PyObject *obj) +{ + /* Modeled after GET_WEAKREFS_LISTPTR(). + + This is never triggered for static types so we can avoid the + (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). */ + PyWeakReference **list = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(obj); + LOCK_WEAKREFS(obj); + while (*list) { + _PyWeakref_ClearRef(*list); } + UNLOCK_WEAKREFS(obj); +} + +int +_PyWeakref_IsDead(PyObject *weakref) +{ + return _PyWeakref_IS_DEAD(weakref); } diff --git a/Python/pystate.c b/Python/pystate.c index 892e740493cdfd..cee481c564b0cb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -506,6 +506,15 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) for (size_t i = 0; i < Py_ARRAY_LENGTH(locks); i++) { _PyMutex_at_fork_reinit(locks[i]); } +#ifdef Py_GIL_DISABLED + for (PyInterpreterState *interp = runtime->interpreters.head; + interp != NULL; interp = interp->next) + { + for (int i = 0; i < NUM_WEAKREF_LIST_LOCKS; i++) { + _PyMutex_at_fork_reinit(&interp->weakref_locks[i]); + } + } +#endif _PyTypes_AfterFork(); From 2067da25796ea3254d0edf61a39bcc0326c4f71d Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 8 Apr 2024 11:53:13 -0400 Subject: [PATCH 114/143] gh-117547: Fix mimalloc compile error on OpenBSD (#117548) --- Objects/mimalloc/prim/unix/prim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c index ec8447ab40d70c..c6ea05bbe7a2ac 100644 --- a/Objects/mimalloc/prim/unix/prim.c +++ b/Objects/mimalloc/prim/unix/prim.c @@ -50,7 +50,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__FreeBSD__) && !defined(__sun) +#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__sun) #define MI_HAS_SYSCALL_H #include #endif @@ -76,7 +76,7 @@ static int mi_prim_access(const char *fpath, int mode) { return syscall(SYS_access,fpath,mode); } -#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__FreeBSD__) && !defined(__sun) // avoid unused warnings +#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__sun) // avoid unused warnings static int mi_prim_open(const char* fpath, int open_flags) { return open(fpath,open_flags); From 1a6594f66166206b08f24c3ba633c85f86f99a56 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 8 Apr 2024 12:11:36 -0400 Subject: [PATCH 115/143] gh-117439: Make refleak checking thread-safe without the GIL (#117469) This keeps track of the per-thread total reference count operations in PyThreadState in the free-threaded builds. The count is merged into the interpreter's total when the thread exits. --- Include/internal/pycore_object.h | 12 +++---- Include/internal/pycore_tstate.h | 4 +++ Objects/bytesobject.c | 2 +- Objects/dictobject.c | 10 +++--- Objects/object.c | 62 +++++++++++++++++--------------- Objects/tupleobject.c | 2 +- Objects/unicodeobject.c | 2 +- Python/gc_free_threading.c | 4 +-- Python/pystate.c | 8 +++++ 9 files changed, 62 insertions(+), 44 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 1e1b664000f108..9aa2e5bf918a7b 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -86,9 +86,9 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( built against the pre-3.12 stable ABI. */ PyAPI_DATA(Py_ssize_t) _Py_RefTotal; -extern void _Py_AddRefTotal(PyInterpreterState *, Py_ssize_t); -extern void _Py_IncRefTotal(PyInterpreterState *); -extern void _Py_DecRefTotal(PyInterpreterState *); +extern void _Py_AddRefTotal(PyThreadState *, Py_ssize_t); +extern void _Py_IncRefTotal(PyThreadState *); +extern void _Py_DecRefTotal(PyThreadState *); # define _Py_DEC_REFTOTAL(interp) \ interp->object_state.reftotal-- @@ -101,7 +101,7 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) return; } #ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyInterpreterState_GET(), n); + _Py_AddRefTotal(_PyThreadState_GET(), n); #endif #if !defined(Py_GIL_DISABLED) op->ob_refcnt += n; @@ -393,7 +393,7 @@ _Py_TryIncrefFast(PyObject *op) { _Py_INCREF_STAT_INC(); _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif return 1; } @@ -416,7 +416,7 @@ _Py_TryIncRefShared(PyObject *op) &shared, shared + (1 << _Py_REF_SHARED_SHIFT))) { #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif _Py_INCREF_STAT_INC(); return 1; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index e268e6fbbb087b..733e3172a1c0ff 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -38,6 +38,10 @@ typedef struct _PyThreadStateImpl { struct _brc_thread_state brc; #endif +#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) + Py_ssize_t reftotal; // this thread's total refcount operations +#endif + } _PyThreadStateImpl; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index d7b0c6b7b01aa9..d576dd93f05e10 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3118,7 +3118,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) PyObject_Realloc(v, PyBytesObject_SIZE + newsize); if (*pv == NULL) { #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif PyObject_Free(v); PyErr_NoMemory(); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 9218b1aa470663..e7993e4b051433 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -445,7 +445,7 @@ dictkeys_incref(PyDictKeysObject *dk) return; } #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif INCREF_KEYS(dk); } @@ -458,7 +458,7 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) } assert(dk->dk_refcnt > 0); #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif if (DECREF_KEYS(dk) == 1) { if (DK_IS_UNICODE(dk)) { @@ -790,7 +790,7 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } } #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; @@ -978,7 +978,7 @@ clone_combined_dict_keys(PyDictObject *orig) we have it now; calling dictkeys_incref would be an error as keys->dk_refcnt is already set to 1 (after memcpy). */ #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif return keys; } @@ -2021,7 +2021,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, if (oldkeys != Py_EMPTY_KEYS) { #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); diff --git a/Objects/object.c b/Objects/object.c index 60642d899bcafa..c8e6f8fc1a2b40 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -73,21 +73,16 @@ get_legacy_reftotal(void) interp->object_state.reftotal static inline void -reftotal_increment(PyInterpreterState *interp) +reftotal_add(PyThreadState *tstate, Py_ssize_t n) { - REFTOTAL(interp)++; -} - -static inline void -reftotal_decrement(PyInterpreterState *interp) -{ - REFTOTAL(interp)--; -} - -static inline void -reftotal_add(PyInterpreterState *interp, Py_ssize_t n) -{ - REFTOTAL(interp) += n; +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + // relaxed store to avoid data race with read in get_reftotal() + Py_ssize_t reftotal = tstate_impl->reftotal + n; + _Py_atomic_store_ssize_relaxed(&tstate_impl->reftotal, reftotal); +#else + REFTOTAL(tstate->interp) += n; +#endif } static inline Py_ssize_t get_global_reftotal(_PyRuntimeState *); @@ -117,7 +112,15 @@ get_reftotal(PyInterpreterState *interp) { /* For a single interpreter, we ignore the legacy _Py_RefTotal, since we can't determine which interpreter updated it. */ - return REFTOTAL(interp); + Py_ssize_t total = REFTOTAL(interp); +#ifdef Py_GIL_DISABLED + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + /* This may race with other threads modifications to their reftotal */ + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)p; + total += _Py_atomic_load_ssize_relaxed(&tstate_impl->reftotal); + } +#endif + return total; } static inline Py_ssize_t @@ -129,7 +132,7 @@ get_global_reftotal(_PyRuntimeState *runtime) HEAD_LOCK(&_PyRuntime); PyInterpreterState *interp = PyInterpreterState_Head(); for (; interp != NULL; interp = PyInterpreterState_Next(interp)) { - total += REFTOTAL(interp); + total += get_reftotal(interp); } HEAD_UNLOCK(&_PyRuntime); @@ -222,32 +225,32 @@ _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op) void _Py_INCREF_IncRefTotal(void) { - reftotal_increment(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), 1); } /* This is used strictly by Py_DECREF(). */ void _Py_DECREF_DecRefTotal(void) { - reftotal_decrement(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), -1); } void -_Py_IncRefTotal(PyInterpreterState *interp) +_Py_IncRefTotal(PyThreadState *tstate) { - reftotal_increment(interp); + reftotal_add(tstate, 1); } void -_Py_DecRefTotal(PyInterpreterState *interp) +_Py_DecRefTotal(PyThreadState *tstate) { - reftotal_decrement(interp); + reftotal_add(tstate, -1); } void -_Py_AddRefTotal(PyInterpreterState *interp, Py_ssize_t n) +_Py_AddRefTotal(PyThreadState *tstate, Py_ssize_t n) { - reftotal_add(interp, n); + reftotal_add(tstate, n); } /* This includes the legacy total @@ -267,7 +270,10 @@ _Py_GetLegacyRefTotal(void) Py_ssize_t _PyInterpreterState_GetRefTotal(PyInterpreterState *interp) { - return get_reftotal(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t total = get_reftotal(interp); + HEAD_UNLOCK(&_PyRuntime); + return total; } #endif /* Py_REF_DEBUG */ @@ -345,7 +351,7 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) if (should_queue) { #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif _Py_brc_queue_object(o); } @@ -405,7 +411,7 @@ _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) &shared, new_shared)); #ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyInterpreterState_GET(), extra); + _Py_AddRefTotal(_PyThreadState_GET(), extra); #endif _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); @@ -2376,7 +2382,7 @@ void _Py_NewReference(PyObject *op) { #ifdef Py_REF_DEBUG - reftotal_increment(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif new_reference(op); } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index d9dc00da368a84..5ae1ee9a89af84 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -946,7 +946,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) if (sv == NULL) { *pv = NULL; #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif PyObject_GC_Del(v); return -1; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e135638c696fa4..59b350f0a609a6 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14916,7 +14916,7 @@ _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) decrements to these objects will not be registered so they need to be accounted for in here. */ for (Py_ssize_t i = 0; i < Py_REFCNT(s) - 2; i++) { - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); } #endif _Py_SetImmortal(s); diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 7e4137a8e342b1..111632ffb77641 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -168,7 +168,7 @@ merge_refcount(PyObject *op, Py_ssize_t extra) refcount += extra; #ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyInterpreterState_GET(), extra); + _Py_AddRefTotal(_PyThreadState_GET(), extra); #endif // No atomics necessary; all other threads in this interpreter are paused. @@ -307,7 +307,7 @@ merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) // decref and deallocate the object once we start the world again. op->ob_ref_shared += (1 << _Py_REF_SHARED_SHIFT); #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif worklist_push(&state->objs_to_decref, op); } diff --git a/Python/pystate.c b/Python/pystate.c index cee481c564b0cb..4a52f6444ba10a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1698,6 +1698,14 @@ tstate_delete_common(PyThreadState *tstate) decrement_stoptheworld_countdown(&runtime->stoptheworld); } } + +#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) + // Add our portion of the total refcount to the interpreter's total. + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + tstate->interp->object_state.reftotal += tstate_impl->reftotal; + tstate_impl->reftotal = 0; +#endif + HEAD_UNLOCK(runtime); #ifdef Py_GIL_DISABLED From 24a2bd048115efae799b0a9c5dd9fbb7a0806978 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Apr 2024 19:27:25 +0300 Subject: [PATCH 116/143] gh-117642: Fix PEP 737 implementation (GH-117643) * Fix implementation of %#T and %#N (they were implemented as %T# and %N#). * Restore tests removed in gh-116417. --- Doc/c-api/unicode.rst | 6 ++-- Doc/whatsnew/3.13.rst | 2 +- Lib/test/test_capi/test_unicode.py | 34 +++++++++++++++++++ ...-04-08-18-53-33.gh-issue-117642._-tYH_.rst | 1 + Objects/unicodeobject.c | 7 ++-- 5 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-04-08-18-53-33.gh-issue-117642._-tYH_.rst diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 78eec14e3a24d6..7320d035bab513 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -523,7 +523,7 @@ APIs: - Get the fully qualified name of an object type; call :c:func:`PyType_GetFullyQualifiedName`. - * - ``T#`` + * - ``#T`` - :c:expr:`PyObject*` - Similar to ``T`` format, but use a colon (``:``) as separator between the module name and the qualified name. @@ -533,7 +533,7 @@ APIs: - Get the fully qualified name of a type; call :c:func:`PyType_GetFullyQualifiedName`. - * - ``N#`` + * - ``#N`` - :c:expr:`PyTypeObject*` - Similar to ``N`` format, but use a colon (``:``) as separator between the module name and the qualified name. @@ -574,7 +574,7 @@ APIs: copied as-is to the result string, and any extra arguments discarded. .. versionchanged:: 3.13 - Support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats added. + Support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats added. .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0fe2dafbfd6f02..72b3a4c951eda6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1775,7 +1775,7 @@ New Features Equivalent to getting the ``type.__module__`` attribute. (Contributed by Eric Snow and Victor Stinner in :gh:`111696`.) -* Add support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats to +* Add support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats to :c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for more information. diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index a64c75c415c3fe..a69f817c515ba7 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -650,6 +650,40 @@ def check_format(expected, format, *args): check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + # test %T + check_format('type: str', + b'type: %T', py_object("abc")) + check_format(f'type: st', + b'type: %.2T', py_object("abc")) + check_format(f'type: str', + b'type: %10T', py_object("abc")) + + class LocalType: + pass + obj = LocalType() + fullname = f'{__name__}.{LocalType.__qualname__}' + check_format(f'type: {fullname}', + b'type: %T', py_object(obj)) + fullname_alt = f'{__name__}:{LocalType.__qualname__}' + check_format(f'type: {fullname_alt}', + b'type: %#T', py_object(obj)) + + # test %N + check_format('type: str', + b'type: %N', py_object(str)) + check_format(f'type: st', + b'type: %.2N', py_object(str)) + check_format(f'type: str', + b'type: %10N', py_object(str)) + + check_format(f'type: {fullname}', + b'type: %N', py_object(type(obj))) + check_format(f'type: {fullname_alt}', + b'type: %#N', py_object(type(obj))) + with self.assertRaisesRegex(TypeError, "%N argument must be a type"): + check_format('type: str', + b'type: %N', py_object("abc")) + # test variable width and precision check_format(' abc', b'%*s', c_int(5), b'abc') check_format('ab', b'%.*s', c_int(2), b'abc') diff --git a/Misc/NEWS.d/next/C API/2024-04-08-18-53-33.gh-issue-117642._-tYH_.rst b/Misc/NEWS.d/next/C API/2024-04-08-18-53-33.gh-issue-117642._-tYH_.rst new file mode 100644 index 00000000000000..edef2777717014 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-04-08-18-53-33.gh-issue-117642._-tYH_.rst @@ -0,0 +1 @@ +Fix :pep:`737` implementation for ``%#T`` and ``%#N``. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 59b350f0a609a6..5f15071d7d54ef 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -2468,6 +2468,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, switch (*f++) { case '-': flags |= F_LJUST; continue; case '0': flags |= F_ZERO; continue; + case '#': flags |= F_ALT; continue; } f--; break; @@ -2797,9 +2798,8 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, PyTypeObject *type = (PyTypeObject *)Py_NewRef(Py_TYPE(obj)); PyObject *type_name; - if (f[1] == '#') { + if (flags & F_ALT) { type_name = _PyType_GetFullyQualifiedName(type, ':'); - f++; } else { type_name = PyType_GetFullyQualifiedName(type); @@ -2830,9 +2830,8 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, PyTypeObject *type = (PyTypeObject*)type_raw; PyObject *type_name; - if (f[1] == '#') { + if (flags & F_ALT) { type_name = _PyType_GetFullyQualifiedName(type, ':'); - f++; } else { type_name = PyType_GetFullyQualifiedName(type); From 775912a51d6847b0e4fe415fa91f2e0b06a3c43c Mon Sep 17 00:00:00 2001 From: Bruce Merry <1963944+bmerry@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:58:02 +0200 Subject: [PATCH 117/143] gh-81322: support multiple separators in StreamReader.readuntil (#16429) --- Doc/library/asyncio-stream.rst | 11 ++++ Lib/asyncio/streams.py | 65 +++++++++++++------ Lib/test/test_asyncio/test_streams.py | 46 +++++++++++++ .../2019-09-26-17-52-52.bpo-37141.onYY2-.rst | 2 + 4 files changed, 103 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-09-26-17-52-52.bpo-37141.onYY2-.rst diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 68b1dff20213e1..6231b49b1e2431 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -260,8 +260,19 @@ StreamReader buffer is reset. The :attr:`IncompleteReadError.partial` attribute may contain a portion of the separator. + The *separator* may also be an :term:`iterable` of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of :exc:`LimitOverrunError`, + the shortest possible separator is considered to be the one that + matched. + .. versionadded:: 3.5.2 + .. versionchanged:: 3.13 + + The *separator* parameter may now be an :term:`iterable` of + separators. + .. method:: at_eof() Return ``True`` if the buffer is empty and :meth:`feed_eof` diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 3fe52dbac25c91..4517ca22d74637 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -590,20 +590,34 @@ async def readuntil(self, separator=b'\n'): If the data cannot be read because of over limit, a LimitOverrunError exception will be raised, and the data will be left in the internal buffer, so it can be read again. + + The ``separator`` may also be an iterable of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of LimitOverrunError, + the shortest possible separator is considered to be the one that + matched. """ - seplen = len(separator) - if seplen == 0: + if isinstance(separator, bytes): + separator = [separator] + else: + # Makes sure shortest matches wins, and supports arbitrary iterables + separator = sorted(separator, key=len) + if not separator: + raise ValueError('Separator should contain at least one element') + min_seplen = len(separator[0]) + max_seplen = len(separator[-1]) + if min_seplen == 0: raise ValueError('Separator should be at least one-byte string') if self._exception is not None: raise self._exception # Consume whole buffer except last bytes, which length is - # one less than seplen. Let's check corner cases with - # separator='SEPARATOR': + # one less than max_seplen. Let's check corner cases with + # separator[-1]='SEPARATOR': # * we have received almost complete separator (without last # byte). i.e buffer='some textSEPARATO'. In this case we - # can safely consume len(separator) - 1 bytes. + # can safely consume max_seplen - 1 bytes. # * last byte of buffer is first byte of separator, i.e. # buffer='abcdefghijklmnopqrS'. We may safely consume # everything except that last byte, but this require to @@ -616,26 +630,35 @@ async def readuntil(self, separator=b'\n'): # messages :) # `offset` is the number of bytes from the beginning of the buffer - # where there is no occurrence of `separator`. + # where there is no occurrence of any `separator`. offset = 0 - # Loop until we find `separator` in the buffer, exceed the buffer size, + # Loop until we find a `separator` in the buffer, exceed the buffer size, # or an EOF has happened. while True: buflen = len(self._buffer) - # Check if we now have enough data in the buffer for `separator` to - # fit. - if buflen - offset >= seplen: - isep = self._buffer.find(separator, offset) - - if isep != -1: - # `separator` is in the buffer. `isep` will be used later - # to retrieve the data. + # Check if we now have enough data in the buffer for shortest + # separator to fit. + if buflen - offset >= min_seplen: + match_start = None + match_end = None + for sep in separator: + isep = self._buffer.find(sep, offset) + + if isep != -1: + # `separator` is in the buffer. `match_start` and + # `match_end` will be used later to retrieve the + # data. + end = isep + len(sep) + if match_end is None or end < match_end: + match_end = end + match_start = isep + if match_end is not None: break # see upper comment for explanation. - offset = buflen + 1 - seplen + offset = max(0, buflen + 1 - max_seplen) if offset > self._limit: raise exceptions.LimitOverrunError( 'Separator is not found, and chunk exceed the limit', @@ -644,7 +667,7 @@ async def readuntil(self, separator=b'\n'): # Complete message (with full separator) may be present in buffer # even when EOF flag is set. This may happen when the last chunk # adds data which makes separator be found. That's why we check for - # EOF *ater* inspecting the buffer. + # EOF *after* inspecting the buffer. if self._eof: chunk = bytes(self._buffer) self._buffer.clear() @@ -653,12 +676,12 @@ async def readuntil(self, separator=b'\n'): # _wait_for_data() will resume reading if stream was paused. await self._wait_for_data('readuntil') - if isep > self._limit: + if match_start > self._limit: raise exceptions.LimitOverrunError( - 'Separator is found, but chunk is longer than limit', isep) + 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:isep + seplen] - del self._buffer[:isep + seplen] + chunk = self._buffer[:match_end] + del self._buffer[:match_end] self._maybe_resume_transport() return bytes(chunk) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 2cf48538d5d30d..792e88761acdc2 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -383,6 +383,10 @@ def test_readuntil_separator(self): stream = asyncio.StreamReader(loop=self.loop) with self.assertRaisesRegex(ValueError, 'Separator should be'): self.loop.run_until_complete(stream.readuntil(separator=b'')) + with self.assertRaisesRegex(ValueError, 'Separator should be'): + self.loop.run_until_complete(stream.readuntil(separator=[b''])) + with self.assertRaisesRegex(ValueError, 'Separator should contain'): + self.loop.run_until_complete(stream.readuntil(separator=[])) def test_readuntil_multi_chunks(self): stream = asyncio.StreamReader(loop=self.loop) @@ -466,6 +470,48 @@ def test_readuntil_limit_found_sep(self): self.assertEqual(b'some dataAAA', stream._buffer) + def test_readuntil_multi_separator(self): + stream = asyncio.StreamReader(loop=self.loop) + + # Simple case + stream.feed_data(b'line 1\nline 2\r') + data = self.loop.run_until_complete(stream.readuntil([b'\r', b'\n'])) + self.assertEqual(b'line 1\n', data) + data = self.loop.run_until_complete(stream.readuntil([b'\r', b'\n'])) + self.assertEqual(b'line 2\r', data) + self.assertEqual(b'', stream._buffer) + + # First end position matches, even if that's a longer match + stream.feed_data(b'ABCDEFG') + data = self.loop.run_until_complete(stream.readuntil([b'DEF', b'BCDE'])) + self.assertEqual(b'ABCDE', data) + self.assertEqual(b'FG', stream._buffer) + + def test_readuntil_multi_separator_limit(self): + stream = asyncio.StreamReader(loop=self.loop, limit=3) + stream.feed_data(b'some dataA') + + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'is found') as cm: + self.loop.run_until_complete(stream.readuntil([b'A', b'ome dataA'])) + + self.assertEqual(b'some dataA', stream._buffer) + + def test_readuntil_multi_separator_negative_offset(self): + # If the buffer is big enough for the smallest separator (but does + # not contain it) but too small for the largest, `offset` must not + # become negative. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'data') + + readuntil_task = self.loop.create_task(stream.readuntil([b'A', b'long sep'])) + self.loop.call_soon(stream.feed_data, b'Z') + self.loop.call_soon(stream.feed_data, b'Aaaa') + + data = self.loop.run_until_complete(readuntil_task) + self.assertEqual(b'dataZA', data) + self.assertEqual(b'aaa', stream._buffer) + def test_readexactly_zero_or_less(self): # Read exact number of bytes (zero or less). stream = asyncio.StreamReader(loop=self.loop) diff --git a/Misc/NEWS.d/next/Library/2019-09-26-17-52-52.bpo-37141.onYY2-.rst b/Misc/NEWS.d/next/Library/2019-09-26-17-52-52.bpo-37141.onYY2-.rst new file mode 100644 index 00000000000000..d916f319947dc4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-26-17-52-52.bpo-37141.onYY2-.rst @@ -0,0 +1,2 @@ +Accept an iterable of separators in :meth:`asyncio.StreamReader.readuntil`, stopping +when one of them is encountered. From ed785c08993467461711c56eb5e6f88331062cca Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Apr 2024 19:16:43 +0200 Subject: [PATCH 118/143] Enhance regrtest get_signal_name(): support shell exit code (#117647) --- Lib/test/libregrtest/utils.py | 8 ++++++++ Lib/test/test_regrtest.py | 1 + 2 files changed, 9 insertions(+) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 837f73b28b4018..791f996127ea58 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -698,6 +698,14 @@ def get_signal_name(exitcode): except ValueError: pass + # Shell exit code (ex: WASI build) + if 128 < exitcode < 256: + signum = exitcode - 128 + try: + return signal.Signals(signum).name + except ValueError: + pass + try: return WINDOWS_STATUS[exitcode] except KeyError: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index d222b3803fdba7..809abd7e92d65f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2291,6 +2291,7 @@ def test_get_signal_name(self): for exitcode, expected in ( (-int(signal.SIGINT), 'SIGINT'), (-int(signal.SIGSEGV), 'SIGSEGV'), + (128 + int(signal.SIGABRT), 'SIGABRT'), (3221225477, "STATUS_ACCESS_VIOLATION"), (0xC00000FD, "STATUS_STACK_OVERFLOW"), ): From ac45766673b181ace8fbafe36c89bde910968f0e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Apr 2024 21:11:08 +0200 Subject: [PATCH 119/143] gh-117645: Skip test_dynamic global specialization on WASI (#117646) Skip test_load_global_specialization_failure_keeps_oparg() of test_dynamic on WASI build. The test uses too much stack memory. --- Lib/test/test_dynamic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 3928bbab4423c2..0cb56a98f1c12a 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -4,7 +4,7 @@ import sys import unittest -from test.support import is_wasi, Py_DEBUG, swap_item, swap_attr +from test.support import is_wasi, swap_item, swap_attr class RebindBuiltinsTests(unittest.TestCase): @@ -134,7 +134,7 @@ def test_eval_gives_lambda_custom_globals(self): self.assertEqual(foo(), 7) - @unittest.skipIf(is_wasi and Py_DEBUG, "stack depth too shallow in pydebug WASI") + @unittest.skipIf(is_wasi, "stack depth too shallow in WASI") def test_load_global_specialization_failure_keeps_oparg(self): # https://github.com/python/cpython/issues/91625 class MyGlobals(dict): From 19a22020676a599e1c92a24f841196645ddd9895 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 8 Apr 2024 23:08:48 -0400 Subject: [PATCH 120/143] gh-117182: Allow lazily loaded modules to modify their own __class__ --- Lib/importlib/util.py | 12 +++++--- Lib/test/test_importlib/test_lazy.py | 28 +++++++++++++++++++ ...-03-23-12-28-05.gh-issue-117182.a0KANW.rst | 2 ++ 3 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-23-12-28-05.gh-issue-117182.a0KANW.rst diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index f1bb4b1fb41576..c94a148e4c50e0 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -178,15 +178,17 @@ def __getattribute__(self, attr): # Only the first thread to get the lock should trigger the load # and reset the module's class. The rest can now getattr(). if object.__getattribute__(self, '__class__') is _LazyModule: + __class__ = loader_state['__class__'] + # Reentrant calls from the same thread must be allowed to proceed without # triggering the load again. # exec_module() and self-referential imports are the primary ways this can # happen, but in any case we must return something to avoid deadlock. if loader_state['is_loading']: - return object.__getattribute__(self, attr) + return __class__.__getattribute__(self, attr) loader_state['is_loading'] = True - __dict__ = object.__getattribute__(self, '__dict__') + __dict__ = __class__.__getattribute__(self, '__dict__') # All module metadata must be gathered from __spec__ in order to avoid # using mutated values. @@ -216,8 +218,10 @@ def __getattribute__(self, attr): # Update after loading since that's what would happen in an eager # loading situation. __dict__.update(attrs_updated) - # Finally, stop triggering this method. - self.__class__ = types.ModuleType + # Finally, stop triggering this method, if the module did not + # already update its own __class__. + if isinstance(self, _LazyModule): + object.__setattr__(self, '__class__', __class__) return getattr(self, attr) diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index 4d2cc4eb62b67c..5c6e0303528906 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -196,6 +196,34 @@ def test_lazy_self_referential_modules(self): test_load = module.loads('{}') self.assertEqual(test_load, {}) + def test_lazy_module_type_override(self): + # Verify that lazy loading works with a module that modifies + # its __class__ to be a custom type. + + # Example module from PEP 726 + module = self.new_module(source_code="""\ +import sys +from types import ModuleType + +CONSTANT = 3.14 + +class ImmutableModule(ModuleType): + def __setattr__(self, name, value): + raise AttributeError('Read-only attribute!') + + def __delattr__(self, name): + raise AttributeError('Read-only attribute!') + +sys.modules[__name__].__class__ = ImmutableModule +""") + sys.modules[TestingImporter.module_name] = module + self.assertIsInstance(module, util._LazyModule) + self.assertEqual(module.CONSTANT, 3.14) + with self.assertRaises(AttributeError): + module.CONSTANT = 2.71 + with self.assertRaises(AttributeError): + del module.CONSTANT + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2024-03-23-12-28-05.gh-issue-117182.a0KANW.rst b/Misc/NEWS.d/next/Library/2024-03-23-12-28-05.gh-issue-117182.a0KANW.rst new file mode 100644 index 00000000000000..6b3b841d9d5d7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-23-12-28-05.gh-issue-117182.a0KANW.rst @@ -0,0 +1,2 @@ +Lazy-loading of modules that modify their own ``__class__`` no longer +reverts the ``__class__`` to :class:`types.ModuleType`. From 99852d9e65aef11fed4bb7bd064e2218220f1ac9 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 9 Apr 2024 10:27:14 +0200 Subject: [PATCH 121/143] gh-117648: Improve performance of os.join (#117654) Replace map() with a method call in the loop body. Co-authored-by: Pieter Eendebak --- Lib/ntpath.py | 2 +- Lib/posixpath.py | 3 ++- .../2024-04-08-20-26-15.gh-issue-117648.NzVEa7.rst | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-08-20-26-15.gh-issue-117648.NzVEa7.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index f9f6c78566e8ed..da5231ff2c0931 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -111,7 +111,7 @@ def join(path, *paths): if not paths: path[:0] + sep #23780: Ensure compatible data type even if p is null. result_drive, result_root, result_path = splitroot(path) - for p in map(os.fspath, paths): + for p in paths: p_drive, p_root, p_path = splitroot(p) if p_root: # Second path is absolute diff --git a/Lib/posixpath.py b/Lib/posixpath.py index b7fbdff20cac99..79e65587e66282 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -79,7 +79,8 @@ def join(a, *p): try: if not p: path[:0] + sep #23780: Ensure compatible data type even if p is null. - for b in map(os.fspath, p): + for b in p: + b = os.fspath(b) if b.startswith(sep): path = b elif not path or path.endswith(sep): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-08-20-26-15.gh-issue-117648.NzVEa7.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-08-20-26-15.gh-issue-117648.NzVEa7.rst new file mode 100644 index 00000000000000..c7e0dfcc461fc9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-08-20-26-15.gh-issue-117648.NzVEa7.rst @@ -0,0 +1 @@ +Speedup :func:`os.path.join` by up to 6% on Windows. From 57183241af76bf33e44d886a733f799d20fc680c Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 9 Apr 2024 01:54:28 -0700 Subject: [PATCH 122/143] gh-107674: Remove some unnecessary code in instrumentation code (GH-117393) --- Python/instrumentation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 0f60290865000c..3866144a19bf74 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1197,7 +1197,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, /* Special case sys.settrace to avoid boxing the line number, * only to immediately unbox it. */ if (tools & (1 << PY_MONITORING_SYS_TRACE_ID)) { - if (tstate->c_tracefunc != NULL && line >= 0) { + if (tstate->c_tracefunc != NULL) { PyFrameObject *frame_obj = _PyFrame_GetFrameObject(frame); if (frame_obj == NULL) { return -1; From 57aee2a02ce38be30cf6c310245547cf56562ab6 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 9 Apr 2024 11:52:31 +0200 Subject: [PATCH 123/143] Python 3.13.0a6 --- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 75 +- Misc/NEWS.d/3.13.0a6.rst | 1270 +++++++++++++++++ ...4-03-06-17-26-55.gh-issue-71052.vLbu9u.rst | 1 - ...-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst | 1 - ...-03-13-16-16-43.gh-issue-114736.ZhmauG.rst | 1 - ...-12-12-19-48-31.gh-issue-113024.rXcQs7.rst | 1 - ...-02-28-15-50-01.gh-issue-111140.mpwcUg.rst | 3 - ...4-03-14-10-33-58.gh-issue-85283.LOgmdU.rst | 3 - ...-03-14-15-17-11.gh-issue-111696.YmnvAi.rst | 4 - ...-03-14-18-00-32.gh-issue-111696.L6oIPq.rst | 3 - ...-03-14-22-30-07.gh-issue-111696.76UMKi.rst | 4 - ...-03-15-23-55-24.gh-issue-115754.xnzc__.rst | 3 - ...-03-15-23-57-33.gh-issue-115754.zLdv82.rst | 5 - ...-03-16-12-21-00.gh-issue-116809.JL786L.rst | 2 - ...-03-17-22-42-21.gh-issue-116936.tNrzfm.rst | 2 - ...-03-18-09-58-46.gh-issue-116869.LFDVKM.rst | 2 - ...-03-18-10-58-47.gh-issue-116869.lN0GBl.rst | 2 - ...-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst | 3 - ...-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst | 2 - ...4-03-22-19-29-24.gh-issue-87193.u7O-jY.rst | 3 - ...-04-08-18-53-33.gh-issue-117642._-tYH_.rst | 1 - .../2021-09-04-22-33-01.bpo-24612.SsTuUX.rst | 2 - ...2-10-05-09-33-48.gh-issue-97901.BOLluU.rst | 1 - ...-10-14-00-05-17.gh-issue-109870.oKpJ3P.rst | 3 - ...-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst | 12 - ...-02-24-03-39-09.gh-issue-115776.THJXqg.rst | 4 - ...-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst | 3 - ...-03-07-16-12-39.gh-issue-114099.ujdjn2.rst | 2 - ...-03-11-00-45-39.gh-issue-116554.gYumG5.rst | 1 - ...-03-11-22-05-56.gh-issue-116626.GsyczB.rst | 1 - ...-03-12-20-31-57.gh-issue-113964.bJppzg.rst | 2 - ...-03-13-16-55-25.gh-issue-116735.o3w6y8.rst | 1 - ...-03-21-09-57-57.gh-issue-117114.Qu-p55.rst | 1 - ...-03-21-12-10-11.gh-issue-117108._6jIrB.rst | 3 - ...-03-25-12-51-12.gh-issue-117108.tNqDEo.rst | 3 - ...4-03-25-17-04-54.gh-issue-99108.8bjdO6.rst | 6 - ...-03-26-17-22-38.gh-issue-117266.Kwh79O.rst | 2 - ...-03-28-19-13-20.gh-issue-117335.d6uKJu.rst | 1 - ...-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst | 1 - ...-03-29-21-43-19.gh-issue-117381.fT0JFM.rst | 1 - ...-04-02-06-16-49.gh-issue-109120.X485oN.rst | 2 - ...-04-02-10-04-57.gh-issue-117411.YdyVmG.rst | 1 - ...-04-02-17-37-35.gh-issue-117431.vDKAOn.rst | 10 - ...-04-03-09-49-15.gh-issue-117431.WAqRgc.rst | 6 - ...-04-03-13-44-04.gh-issue-116968.zgcdG2.rst | 11 - ...-04-04-13-42-59.gh-issue-117494.GPQH64.rst | 1 - ...-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst | 1 - ...-04-08-20-26-15.gh-issue-117648.NzVEa7.rst | 1 - ...2-04-15-13-15-23.gh-issue-91565.OznXwC.rst | 1 - ...-03-20-12-41-47.gh-issue-114099.ad_Ck9.rst | 1 - ...-03-20-15-12-37.gh-issue-115977.IMLi6K.rst | 1 - .../2019-08-12-19-08-06.bpo-15010.3bY2CF.rst | 3 - ...9-08-27-01-03-26.gh-issue-66543._TRpYr.rst | 4 - .../2019-09-26-17-52-52.bpo-37141.onYY2-.rst | 2 - .../2020-06-11-16-20-33.bpo-27578.CIA-fu.rst | 3 - .../2020-10-02-17-35-19.bpo-33533.GLIhM5.rst | 5 - ...2-06-22-14-45-32.gh-issue-89739.CqZcRL.rst | 1 - ...3-05-06-05-00-42.gh-issue-96471.S3X5I-.rst | 2 - ...-06-16-19-17-06.gh-issue-105866.0NBveV.rst | 1 - ...-12-11-00-51-51.gh-issue-112948.k-OKp5.rst | 1 - ...-12-28-22-52-45.gh-issue-113548.j6TJ7O.rst | 1 - ...4-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst | 2 - ...-01-22-15-50-58.gh-issue-113538.v2wrwg.rst | 3 - ...-02-01-03-09-38.gh-issue-114271.raCkt5.rst | 7 - ...-02-01-08-09-20.gh-issue-114847.-JrWrR.rst | 1 - ...-02-18-09-50-31.gh-issue-115627.HGchj0.rst | 2 - ...-02-26-10-06-50.gh-issue-113308.MbvOFt.rst | 4 - ...4-03-01-20-23-57.gh-issue-90535.wXm-jC.rst | 3 - ...4-03-05-19-56-29.gh-issue-71052.PMDK--.rst | 1 - ...-03-06-18-30-37.gh-issue-116401.3Wcda2.rst | 2 - ...-03-07-11-10-27.gh-issue-114314.iEhAMH.rst | 3 - ...-03-08-11-31-49.gh-issue-116484.VMAsU7.rst | 3 - ...-03-11-17-04-55.gh-issue-116608.30f58-.rst | 10 - ...4-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst | 2 - ...4-03-12-19-32-17.gh-issue-71042.oI0Ron.rst | 2 - ...4-03-13-15-45-54.gh-issue-63283.OToJnG.rst | 2 - ...-03-14-01-38-44.gh-issue-113171.VFnObz.rst | 9 - ...-03-14-09-38-51.gh-issue-116647.h0d_zj.rst | 1 - ...-03-14-10-01-23.gh-issue-116811._h5iKP.rst | 2 - ...-03-14-14-01-46.gh-issue-116764.moB3Lc.rst | 4 - ...4-03-14-17-21-25.gh-issue-63207.LV16SL.rst | 4 - ...-03-14-17-24-59.gh-issue-106531.9ehywi.rst | 5 - ...4-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst | 1 - ...-03-17-18-12-39.gh-issue-115538.PBiRQB.rst | 2 - ...-03-18-14-36-50.gh-issue-116957.dTCs4f.rst | 3 - ...4-03-19-11-08-26.gh-issue-90872.ghys95.rst | 3 - ...-03-19-14-35-57.gh-issue-114099.siNSpK.rst | 1 - ...-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst | 1 - ...4-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst | 7 - ...-03-20-23-07-58.gh-issue-109653.uu3lrX.rst | 2 - ...-03-21-07-27-36.gh-issue-117110.9K1InX.rst | 1 - ...-03-21-17-07-38.gh-issue-117084.w1mTpT.rst | 2 - ...-03-23-12-28-05.gh-issue-117182.a0KANW.rst | 2 - ...-03-23-13-40-13.gh-issue-112383.XuHf3G.rst | 1 - ...-03-23-14-26-18.gh-issue-117178.vTisTG.rst | 2 - ...-03-25-00-20-16.gh-issue-117205.yV7xGb.rst | 1 - ...-03-25-21-15-56.gh-issue-117225.oOaZXb.rst | 2 - ...4-03-26-11-48-39.gh-issue-98966.SayV9y.rst | 2 - ...-03-27-16-43-42.gh-issue-117294.wbXNFv.rst | 2 - ...-03-27-21-05-52.gh-issue-117310.Bt2wox.rst | 4 - ...4-03-28-13-54-20.gh-issue-88014.zJz31I.rst | 3 - ...4-03-28-17-55-22.gh-issue-66449.4jhuEV.rst | 2 - ...-03-29-12-07-26.gh-issue-117348.WjCYvK.rst | 2 - ...-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst | 3 - ...-04-02-13-13-46.gh-issue-117459.jiIZmH.rst | 1 - ...-04-02-20-30-12.gh-issue-114848.YX4pEc.rst | 2 - ...-04-03-18-36-53.gh-issue-117467.l6rWlj.rst | 2 - ...-03-06-11-00-36.gh-issue-116307.Uij0t_.rst | 3 - ...-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst | 2 - ...-03-13-12-06-49.gh-issue-115979.zsNpQD.rst | 1 - ...-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst | 1 - ...-03-21-11-32-29.gh-issue-116333.F-9Ram.rst | 3 - ...-03-24-23-49-25.gh-issue-117187.eMLT5n.rst | 1 - ...4-03-25-21-31-49.gh-issue-83434.U7Z8cY.rst | 3 - ...-02-08-14-48-15.gh-issue-115119.qMt32O.rst | 3 - ...4-02-24-23-03-43.gh-issue-91227.sL4zWC.rst | 1 - ...-03-14-01-58-22.gh-issue-116773.H2UldY.rst | 1 - ...4-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst | 4 - ...-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst | 1 - ...-03-28-22-12-00.gh-issue-117267.K_tki1.rst | 5 - README.rst | 2 +- 122 files changed, 1315 insertions(+), 346 deletions(-) create mode 100644 Misc/NEWS.d/3.13.0a6.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-03-06-17-26-55.gh-issue-71052.vLbu9u.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst delete mode 100644 Misc/NEWS.d/next/Build/2024-03-13-16-16-43.gh-issue-114736.ZhmauG.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-12-12-19-48-31.gh-issue-113024.rXcQs7.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-02-28-15-50-01.gh-issue-111140.mpwcUg.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-14-15-17-11.gh-issue-111696.YmnvAi.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-14-18-00-32.gh-issue-111696.L6oIPq.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-14-22-30-07.gh-issue-111696.76UMKi.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-15-23-55-24.gh-issue-115754.xnzc__.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-15-23-57-33.gh-issue-115754.zLdv82.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-16-12-21-00.gh-issue-116809.JL786L.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-17-22-42-21.gh-issue-116936.tNrzfm.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-03-22-19-29-24.gh-issue-87193.u7O-jY.rst delete mode 100644 Misc/NEWS.d/next/C API/2024-04-08-18-53-33.gh-issue-117642._-tYH_.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-09-04-22-33-01.bpo-24612.SsTuUX.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-05-09-33-48.gh-issue-97901.BOLluU.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-14-00-05-17.gh-issue-109870.oKpJ3P.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-13-16-55-25.gh-issue-116735.o3w6y8.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-21-09-57-57.gh-issue-117114.Qu-p55.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-21-12-10-11.gh-issue-117108._6jIrB.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-25-12-51-12.gh-issue-117108.tNqDEo.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-25-17-04-54.gh-issue-99108.8bjdO6.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-26-17-22-38.gh-issue-117266.Kwh79O.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-28-19-13-20.gh-issue-117335.d6uKJu.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-29-21-43-19.gh-issue-117381.fT0JFM.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-02-06-16-49.gh-issue-109120.X485oN.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-02-10-04-57.gh-issue-117411.YdyVmG.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-03-09-49-15.gh-issue-117431.WAqRgc.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-03-13-44-04.gh-issue-116968.zgcdG2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-04-13-42-59.gh-issue-117494.GPQH64.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-08-20-26-15.gh-issue-117648.NzVEa7.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2022-04-15-13-15-23.gh-issue-91565.OznXwC.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2024-03-20-12-41-47.gh-issue-114099.ad_Ck9.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2024-03-20-15-12-37.gh-issue-115977.IMLi6K.rst delete mode 100644 Misc/NEWS.d/next/Library/2019-08-12-19-08-06.bpo-15010.3bY2CF.rst delete mode 100644 Misc/NEWS.d/next/Library/2019-08-27-01-03-26.gh-issue-66543._TRpYr.rst delete mode 100644 Misc/NEWS.d/next/Library/2019-09-26-17-52-52.bpo-37141.onYY2-.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-02-17-35-19.bpo-33533.GLIhM5.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-06-22-14-45-32.gh-issue-89739.CqZcRL.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-06-05-00-42.gh-issue-96471.S3X5I-.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-06-16-19-17-06.gh-issue-105866.0NBveV.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-12-11-00-51-51.gh-issue-112948.k-OKp5.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-12-28-22-52-45.gh-issue-113548.j6TJ7O.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-01-03-09-38.gh-issue-114271.raCkt5.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-01-08-09-20.gh-issue-114847.-JrWrR.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-18-09-50-31.gh-issue-115627.HGchj0.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-02-26-10-06-50.gh-issue-113308.MbvOFt.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-01-20-23-57.gh-issue-90535.wXm-jC.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-05-19-56-29.gh-issue-71052.PMDK--.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-07-11-10-27.gh-issue-114314.iEhAMH.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-08-11-31-49.gh-issue-116484.VMAsU7.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-11-17-04-55.gh-issue-116608.30f58-.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-12-19-32-17.gh-issue-71042.oI0Ron.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-13-15-45-54.gh-issue-63283.OToJnG.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-14-09-38-51.gh-issue-116647.h0d_zj.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-14-10-01-23.gh-issue-116811._h5iKP.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-14-14-01-46.gh-issue-116764.moB3Lc.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-14-17-24-59.gh-issue-106531.9ehywi.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-18-14-36-50.gh-issue-116957.dTCs4f.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-19-11-08-26.gh-issue-90872.ghys95.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-19-14-35-57.gh-issue-114099.siNSpK.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-20-23-07-58.gh-issue-109653.uu3lrX.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-21-07-27-36.gh-issue-117110.9K1InX.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-21-17-07-38.gh-issue-117084.w1mTpT.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-23-12-28-05.gh-issue-117182.a0KANW.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-23-14-26-18.gh-issue-117178.vTisTG.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-25-00-20-16.gh-issue-117205.yV7xGb.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-25-21-15-56.gh-issue-117225.oOaZXb.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-26-11-48-39.gh-issue-98966.SayV9y.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-27-16-43-42.gh-issue-117294.wbXNFv.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-27-21-05-52.gh-issue-117310.Bt2wox.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-28-13-54-20.gh-issue-88014.zJz31I.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-28-17-55-22.gh-issue-66449.4jhuEV.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-29-12-07-26.gh-issue-117348.WjCYvK.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-04-02-13-13-46.gh-issue-117459.jiIZmH.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-04-02-20-30-12.gh-issue-114848.YX4pEc.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-04-03-18-36-53.gh-issue-117467.l6rWlj.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-03-06-11-00-36.gh-issue-116307.Uij0t_.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-03-13-12-06-49.gh-issue-115979.zsNpQD.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-03-21-11-32-29.gh-issue-116333.F-9Ram.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst delete mode 100644 Misc/NEWS.d/next/Tests/2024-03-25-21-31-49.gh-issue-83434.U7Z8cY.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-02-24-23-03-43.gh-issue-91227.sL4zWC.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-03-14-01-58-22.gh-issue-116773.H2UldY.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst delete mode 100644 Misc/NEWS.d/next/Windows/2024-03-28-22-12-00.gh-issue-117267.K_tki1.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 942922bd0df698..c14b1811ad758d 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 13 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 5 +#define PY_RELEASE_SERIAL 6 /* Version as a string */ -#define PY_VERSION "3.13.0a5+" +#define PY_VERSION "3.13.0a6" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 05045ac8c945c8..43c47c22e5c4aa 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Mar 12 18:35:04 2024 +# Autogenerated by Sphinx on Tue Apr 9 11:53:07 2024 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -5221,12 +5221,13 @@ 'the\n' 'current directory, it is read with "\'utf-8\'" encoding and ' 'executed as\n' - 'if it had been typed at the debugger prompt. This is ' - 'particularly\n' - 'useful for aliases. If both files exist, the one in the home\n' - 'directory is read first and aliases defined there can be ' - 'overridden by\n' - 'the local file.\n' + 'if it had been typed at the debugger prompt, with the exception ' + 'that\n' + 'empty lines and lines starting with "#" are ignored. This is\n' + 'particularly useful for aliases. If both files exist, the one ' + 'in the\n' + 'home directory is read first and aliases defined there can be\n' + 'overridden by the local file.\n' '\n' 'Changed in version 3.2: ".pdbrc" can now contain commands that\n' 'continue debugging, such as "continue" or "next". Previously, ' @@ -8640,32 +8641,36 @@ '\n' ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' '\n' - 'The "nonlocal" statement causes the listed identifiers to refer ' - 'to\n' - 'previously bound variables in the nearest enclosing scope ' - 'excluding\n' - 'globals. This is important because the default behavior for ' - 'binding is\n' - 'to search the local namespace first. The statement allows\n' - 'encapsulated code to rebind variables outside of the local ' - 'scope\n' - 'besides the global (module) scope.\n' - '\n' - 'Names listed in a "nonlocal" statement, unlike those listed in ' - 'a\n' - '"global" statement, must refer to pre-existing bindings in an\n' - 'enclosing scope (the scope in which a new binding should be ' - 'created\n' - 'cannot be determined unambiguously).\n' - '\n' - 'Names listed in a "nonlocal" statement must not collide with ' - 'pre-\n' - 'existing bindings in the local scope.\n' + 'When the definition of a function or class is nested (enclosed) ' + 'within\n' + 'the definitions of other functions, its nonlocal scopes are the ' + 'local\n' + 'scopes of the enclosing functions. The "nonlocal" statement ' + 'causes the\n' + 'listed identifiers to refer to names previously bound in ' + 'nonlocal\n' + 'scopes. It allows encapsulated code to rebind such nonlocal\n' + 'identifiers. If a name is bound in more than one nonlocal ' + 'scope, the\n' + 'nearest binding is used. If a name is not bound in any nonlocal ' + 'scope,\n' + 'or if there is no nonlocal scope, a "SyntaxError" is raised.\n' + '\n' + 'The nonlocal statement applies to the entire scope of a function ' + 'or\n' + 'class body. A "SyntaxError" is raised if a variable is used or\n' + 'assigned to prior to its nonlocal declaration in the scope.\n' '\n' 'See also:\n' '\n' ' **PEP 3104** - Access to Names in Outer Scopes\n' - ' The specification for the "nonlocal" statement.\n', + ' The specification for the "nonlocal" statement.\n' + '\n' + '**Programmer’s note:** "nonlocal" is a directive to the parser ' + 'and\n' + 'applies only to code parsed along with it. See the note for ' + 'the\n' + '"global" statement.\n', 'numbers': 'Numeric literals\n' '****************\n' '\n' @@ -13805,14 +13810,18 @@ 'contains\n' 'the numbers 0, 1, …, *n*-1. Item *i* of sequence *a* is selected ' 'by\n' - '"a[i]".\n' + '"a[i]". Some sequences, including built-in sequences, interpret\n' + 'negative subscripts by adding the sequence length. For example,\n' + '"a[-2]" equals "a[n-2]", the second to last item of sequence a ' + 'with\n' + 'length "n".\n' '\n' 'Sequences also support slicing: "a[i:j]" selects all items with ' 'index\n' '*k* such that *i* "<=" *k* "<" *j*. When used as an expression, a\n' - 'slice is a sequence of the same type. This implies that the index ' - 'set\n' - 'is renumbered so that it starts at 0.\n' + 'slice is a sequence of the same type. The comment above about ' + 'negative\n' + 'indexes also applies to negative slice positions.\n' '\n' 'Some sequences also support “extended slicing” with a third “step”\n' 'parameter: "a[i:j:k]" selects all items of *a* with index *x* where ' diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst new file mode 100644 index 00000000000000..f7e90722d659a8 --- /dev/null +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -0,0 +1,1270 @@ +.. date: 2024-04-08-20-26-15 +.. gh-issue: 117648 +.. nonce: NzVEa7 +.. release date: 2024-04-09 +.. section: Core and Builtins + +Speedup :func:`os.path.join` by up to 6% on Windows. + +.. + +.. date: 2024-04-06-16-42-34 +.. gh-issue: 117584 +.. nonce: hqk9Hn +.. section: Core and Builtins + +Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath()`. + +.. + +.. date: 2024-04-04-13-42-59 +.. gh-issue: 117494 +.. nonce: GPQH64 +.. section: Core and Builtins + +Refactored the instruction sequence data structure out of compile.c into +instruction_sequence.c. + +.. + +.. date: 2024-04-03-13-44-04 +.. gh-issue: 116968 +.. nonce: zgcdG2 +.. section: Core and Builtins + +Introduce a unified 16-bit backoff counter type (``_Py_BackoffCounter``), +shared between the Tier 1 adaptive specializer and the Tier 2 optimizer. The +API used for adaptive specialization counters is changed but the behavior is +(supposed to be) identical. + +The behavior of the Tier 2 counters is changed: + +* There are no longer dynamic thresholds (we never varied these). +* All counters now use the same exponential backoff. +* The counter for ``JUMP_BACKWARD`` starts counting down from 16. +* The ``temperature`` in side exits starts counting down from 64. + +.. + +.. date: 2024-04-03-09-49-15 +.. gh-issue: 117431 +.. nonce: WAqRgc +.. section: Core and Builtins + +Improve the performance of the following :class:`bytes` and +:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` +calling convention: + +* :meth:`!endswith` +* :meth:`!startswith` + +.. + +.. date: 2024-04-02-17-37-35 +.. gh-issue: 117431 +.. nonce: vDKAOn +.. section: Core and Builtins + +Improve the performance of the following :class:`str` methods by adapting +them to the :c:macro:`METH_FASTCALL` calling convention: + +* :meth:`~str.count` +* :meth:`~str.endswith` +* :meth:`~str.find` +* :meth:`~str.index` +* :meth:`~str.rfind` +* :meth:`~str.rindex` +* :meth:`~str.startswith` + +.. + +.. date: 2024-04-02-10-04-57 +.. gh-issue: 117411 +.. nonce: YdyVmG +.. section: Core and Builtins + +Move ``PyFutureFeatures`` to an internal header and make it private. + +.. + +.. date: 2024-04-02-06-16-49 +.. gh-issue: 109120 +.. nonce: X485oN +.. section: Core and Builtins + +Added handle of incorrect star expressions, e.g ``f(3, *)``. Patch by +Grigoryev Semyon + +.. + +.. date: 2024-03-29-21-43-19 +.. gh-issue: 117381 +.. nonce: fT0JFM +.. section: Core and Builtins + +Fix error message for :func:`ntpath.commonpath`. + +.. + +.. date: 2024-03-29-15-04-13 +.. gh-issue: 117349 +.. nonce: OB9kQQ +.. section: Core and Builtins + +Optimise several functions in :mod:`os.path`. + +.. + +.. date: 2024-03-28-19-13-20 +.. gh-issue: 117335 +.. nonce: d6uKJu +.. section: Core and Builtins + +Raise TypeError for non-sequences for :func:`ntpath.commonpath`. + +.. + +.. date: 2024-03-26-17-22-38 +.. gh-issue: 117266 +.. nonce: Kwh79O +.. section: Core and Builtins + +Fix crashes for certain user-created subclasses of :class:`ast.AST`. Such +classes are now expected to set the ``_field_types`` attribute. + +.. + +.. date: 2024-03-25-17-04-54 +.. gh-issue: 99108 +.. nonce: 8bjdO6 +.. section: Core and Builtins + +Updated the :mod:`hashlib` built-in `HACL\* project`_ C code from upstream +that we use for many implementations when they are not present via OpenSSL +in a given build. This also avoids the rare potential for a C symbol name +one definition rule linking issue. + +.. _HACL\* project: https://github.com/hacl-star/hacl-star + +.. + +.. date: 2024-03-25-12-51-12 +.. gh-issue: 117108 +.. nonce: tNqDEo +.. section: Core and Builtins + +Change the old space bit of objects in the young generation from 0 to +gcstate->visited, so that any objects created during GC will have the old +bit set correctly if they get moved into the old generation. + +.. + +.. date: 2024-03-21-12-10-11 +.. gh-issue: 117108 +.. nonce: _6jIrB +.. section: Core and Builtins + +The cycle GC now chooses the size of increments based on the total heap +size, instead of the rate of object creation. This ensures that it can keep +up with growing heaps. + +.. + +.. date: 2024-03-21-09-57-57 +.. gh-issue: 117114 +.. nonce: Qu-p55 +.. section: Core and Builtins + +Make :func:`os.path.isdevdrive` available on all platforms. For those that +do not offer Dev Drives, it will always return ``False``. + +.. + +.. date: 2024-03-13-16-55-25 +.. gh-issue: 116735 +.. nonce: o3w6y8 +.. section: Core and Builtins + +For ``INSTRUMENTED_CALL_FUNCTION_EX``, set ``arg0`` to +``sys.monitoring.MISSING`` instead of ``None`` for :monitoring-event:`CALL` +event. + +.. + +.. date: 2024-03-12-20-31-57 +.. gh-issue: 113964 +.. nonce: bJppzg +.. section: Core and Builtins + +Starting new threads and process creation through :func:`os.fork` are now +only prevented once all non-daemon threads exit. + +.. + +.. date: 2024-03-11-22-05-56 +.. gh-issue: 116626 +.. nonce: GsyczB +.. section: Core and Builtins + +Ensure ``INSTRUMENTED_CALL_FUNCTION_EX`` always emits +:monitoring-event:`CALL` + +.. + +.. date: 2024-03-11-00-45-39 +.. gh-issue: 116554 +.. nonce: gYumG5 +.. section: Core and Builtins + +``list.sort()`` now exploits more cases of partial ordering, particularly +those with long descending runs with sub-runs of equal values. Those are +recognized as single runs now (previously, each block of repeated values +caused a new run to be created). + +.. + +.. date: 2024-03-07-16-12-39 +.. gh-issue: 114099 +.. nonce: ujdjn2 +.. section: Core and Builtins + +Added a Loader that can discover extension modules in an iOS-style +Frameworks folder. + +.. + +.. date: 2024-02-25-14-17-25 +.. gh-issue: 115775 +.. nonce: CNbGbJ +.. section: Core and Builtins + +Compiler populates the new ``__static_attributes__`` field on a class with +the names of attributes of this class which are accessed through self.X from +any function in its body. + +.. + +.. date: 2024-02-24-03-39-09 +.. gh-issue: 115776 +.. nonce: THJXqg +.. section: Core and Builtins + +The array of values, the ``PyDictValues`` struct is now embedded in the +object during allocation. This provides better performance in the common +case, and does not degrade as much when the object's ``__dict__`` is +materialized. + +.. + +.. date: 2024-01-07-04-22-51 +.. gh-issue: 108362 +.. nonce: oB9Gcf +.. section: Core and Builtins + +Implement an incremental cyclic garbage collector. By collecting the old +generation in increments, there is no need for a full heap scan. This can +hugely reduce maximum pause time for programs with large heaps. + +Reduce the number of generations from three to two. The old generation is +split into two spaces, "visited" and "pending". + +Collection happens in two steps:: * An increment is formed from the young +generation and a small part of the pending space. * This increment is +scanned and the survivors moved to the end of the visited space. + +When the collecting space becomes empty, the two spaces are swapped. + +.. + +.. date: 2023-10-14-00-05-17 +.. gh-issue: 109870 +.. nonce: oKpJ3P +.. section: Core and Builtins + +Dataclasses now calls :func:`exec` once per dataclass, instead of once per +method being added. This can speed up dataclass creation by up to 20%. + +.. + +.. date: 2022-10-05-09-33-48 +.. gh-issue: 97901 +.. nonce: BOLluU +.. section: Core and Builtins + +Mime type ``text/rtf`` is now supported by :mod:`mimetypes`. + +.. + +.. bpo: 24612 +.. date: 2021-09-04-22-33-01 +.. nonce: SsTuUX +.. section: Core and Builtins + +Improve the :exc:`SyntaxError` that happens when 'not' appears after an +operator. Patch by Pablo Galindo + +.. + +.. date: 2024-04-03-18-36-53 +.. gh-issue: 117467 +.. nonce: l6rWlj +.. section: Library + +Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush`. +Patch by Tony Mountifield. + +.. + +.. date: 2024-04-02-20-30-12 +.. gh-issue: 114848 +.. nonce: YX4pEc +.. section: Library + +Raise :exc:`FileNotFoundError` when ``getcwd()`` returns '(unreachable)', +which can happen on Linux >= 2.6.36 with glibc < 2.27. + +.. + +.. date: 2024-04-02-13-13-46 +.. gh-issue: 117459 +.. nonce: jiIZmH +.. section: Library + +:meth:`asyncio.asyncio.run_coroutine_threadsafe` now keeps the traceback of +:class:`CancelledError`, :class:`TimeoutError` and +:class:`InvalidStateError` which are raised in the coroutine. + +.. + +.. date: 2024-03-29-15-58-01 +.. gh-issue: 117337 +.. nonce: 7w3Qwp +.. section: Library + +Deprecate undocumented :func:`!glob.glob0` and :func:`!glob.glob1` +functions. Use :func:`glob.glob` and pass a directory to its *root_dir* +argument instead. + +.. + +.. date: 2024-03-29-12-07-26 +.. gh-issue: 117348 +.. nonce: WjCYvK +.. section: Library + +Refactored :meth:`configparser.RawConfigParser._read` to reduce cyclometric +complexity and improve comprehensibility. + +.. + +.. date: 2024-03-28-17-55-22 +.. gh-issue: 66449 +.. nonce: 4jhuEV +.. section: Library + +:class:`configparser.ConfigParser` now accepts unnamed sections before named +ones, if configured to do so. + +.. + +.. date: 2024-03-28-13-54-20 +.. gh-issue: 88014 +.. nonce: zJz31I +.. section: Library + +In documentation of :class:`gzip.GzipFile` in module gzip, explain data type +of optional constructor argument *mtime*, and recommend ``mtime = 0`` for +generating deterministic streams. + +.. + +.. date: 2024-03-27-21-05-52 +.. gh-issue: 117310 +.. nonce: Bt2wox +.. section: Library + +Fixed an unlikely early & extra ``Py_DECREF`` triggered crash in :mod:`ssl` +when creating a new ``_ssl._SSLContext`` if CPython was built implausibly +such that the default cipher list is empty **or** the SSL library it was +linked against reports a failure from its C ``SSL_CTX_set_cipher_list()`` +API. + +.. + +.. date: 2024-03-27-16-43-42 +.. gh-issue: 117294 +.. nonce: wbXNFv +.. section: Library + +A ``DocTestCase`` now reports as skipped if all examples in the doctest are +skipped. + +.. + +.. date: 2024-03-26-11-48-39 +.. gh-issue: 98966 +.. nonce: SayV9y +.. section: Library + +In :mod:`subprocess`, raise a more informative message when +``stdout=STDOUT``. + +.. + +.. date: 2024-03-25-21-15-56 +.. gh-issue: 117225 +.. nonce: oOaZXb +.. section: Library + +doctest: only print "and X failed" when non-zero, don't pluralise "1 items". +Patch by Hugo van Kemenade. + +.. + +.. date: 2024-03-25-00-20-16 +.. gh-issue: 117205 +.. nonce: yV7xGb +.. section: Library + +Speed up :func:`compileall.compile_dir` by 20% when using multiprocessing by +increasing ``chunksize``. + +.. + +.. date: 2024-03-23-14-26-18 +.. gh-issue: 117178 +.. nonce: vTisTG +.. section: Library + +Fix regression in lazy loading of self-referential modules, introduced in +gh-114781. + +.. + +.. date: 2024-03-23-13-40-13 +.. gh-issue: 112383 +.. nonce: XuHf3G +.. section: Library + +Fix :mod:`dis` module's handling of ``ENTER_EXECUTOR`` instructions. + +.. + +.. date: 2024-03-23-12-28-05 +.. gh-issue: 117182 +.. nonce: a0KANW +.. section: Library + +Lazy-loading of modules that modify their own ``__class__`` no longer +reverts the ``__class__`` to :class:`types.ModuleType`. + +.. + +.. date: 2024-03-21-17-07-38 +.. gh-issue: 117084 +.. nonce: w1mTpT +.. section: Library + +Fix :mod:`zipfile` extraction for directory entries with the name containing +backslashes on Windows. + +.. + +.. date: 2024-03-21-07-27-36 +.. gh-issue: 117110 +.. nonce: 9K1InX +.. section: Library + +Fix a bug that prevents subclasses of :class:`typing.Any` to be instantiated +with arguments. Patch by Chris Fu. + +.. + +.. date: 2024-03-20-23-07-58 +.. gh-issue: 109653 +.. nonce: uu3lrX +.. section: Library + +Deferred select imports in importlib.metadata and importlib.resources for a +14% speedup. + +.. + +.. date: 2024-03-20-16-10-29 +.. gh-issue: 70647 +.. nonce: FpD6Ar +.. section: Library + +Start the deprecation period for the current behavior of +:func:`datetime.datetime.strptime` and :func:`time.strptime` which always +fails to parse a date string with a :exc:`ValueError` involving a day of +month such as ``strptime("02-29", "%m-%d")`` when a year is **not** +specified and the date happen to be February 29th. This should help avoid +users finding new bugs every four years due to a natural mistaken assumption +about the API when parsing partial date values. + +.. + +.. date: 2024-03-19-19-42-25 +.. gh-issue: 116987 +.. nonce: ZVKUH1 +.. section: Library + +Fixed :func:`inspect.findsource` for class code objects. + +.. + +.. date: 2024-03-19-14-35-57 +.. gh-issue: 114099 +.. nonce: siNSpK +.. section: Library + +Modify standard library to allow for iOS platform differences. + +.. + +.. date: 2024-03-19-11-08-26 +.. gh-issue: 90872 +.. nonce: ghys95 +.. section: Library + +On Windows, :meth:`subprocess.Popen.wait` no longer calls +``WaitForSingleObject()`` with a negative timeout: pass ``0`` ms if the +timeout is negative. Patch by Victor Stinner. + +.. + +.. date: 2024-03-18-14-36-50 +.. gh-issue: 116957 +.. nonce: dTCs4f +.. section: Library + +configparser: Don't leave ConfigParser values in an invalid state (stored as +a list instead of a str) after an earlier read raised DuplicateSectionError +or DuplicateOptionError. + +.. + +.. date: 2024-03-17-18-12-39 +.. gh-issue: 115538 +.. nonce: PBiRQB +.. section: Library + +:class:`_io.WindowsConsoleIO` now emit a warning if a boolean value is +passed as a filedescriptor argument. + +.. + +.. date: 2024-03-14-20-59-28 +.. gh-issue: 90095 +.. nonce: 7UaJ1U +.. section: Library + +Ignore empty lines and comments in ``.pdbrc`` + +.. + +.. date: 2024-03-14-17-24-59 +.. gh-issue: 106531 +.. nonce: 9ehywi +.. section: Library + +Refreshed zipfile._path from `zipp 3.18 +`_, providing +better compatibility for PyPy, better glob performance for deeply nested +zipfiles, and providing internal access to ``CompleteDirs.inject`` for use +in other tests (like importlib.resources). + +.. + +.. date: 2024-03-14-17-21-25 +.. gh-issue: 63207 +.. nonce: LV16SL +.. section: Library + +On Windows, :func:`time.time()` now uses the +``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better than +1 us, instead of the ``GetSystemTimeAsFileTime()`` clock which has a +resolution of 15.6 ms. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-14-01-46 +.. gh-issue: 116764 +.. nonce: moB3Lc +.. section: Library + +Restore support of ``None`` and other false values in :mod:`urllib.parse` +functions :func:`~urllib.parse.parse_qs` and +:func:`~urllib.parse.parse_qsl`. Also, they now raise a TypeError for +non-zero integers and non-empty sequences. + +.. + +.. date: 2024-03-14-10-01-23 +.. gh-issue: 116811 +.. nonce: _h5iKP +.. section: Library + +In ``PathFinder.invalidate_caches``, delegate to +``MetadataPathFinder.invalidate_caches``. + +.. + +.. date: 2024-03-14-09-38-51 +.. gh-issue: 116647 +.. nonce: h0d_zj +.. section: Library + +Fix recursive child in dataclasses + +.. + +.. date: 2024-03-14-01-38-44 +.. gh-issue: 113171 +.. nonce: VFnObz +.. section: Library + +Fixed various false positives and false negatives in + +* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details) +* :attr:`ipaddress.IPv4Address.is_global` +* :attr:`ipaddress.IPv6Address.is_private` +* :attr:`ipaddress.IPv6Address.is_global` + +Also in the corresponding :class:`ipaddress.IPv4Network` and +:class:`ipaddress.IPv6Network` attributes. + +.. + +.. date: 2024-03-13-15-45-54 +.. gh-issue: 63283 +.. nonce: OToJnG +.. section: Library + +In :mod:`encodings.idna`, any capitalization of the the ACE prefix +(``xn--``) is now acceptable. Patch by Pepijn de Vos and Zackery Spytz. + +.. + +.. date: 2024-03-12-19-32-17 +.. gh-issue: 71042 +.. nonce: oI0Ron +.. section: Library + +Add :func:`platform.android_ver`, which provides device and OS information +on Android. + +.. + +.. date: 2024-03-12-17-53-14 +.. gh-issue: 73468 +.. nonce: z4ZzvJ +.. section: Library + +Added new :func:`math.fma` function, wrapping C99's ``fma()`` operation: +fused multiply-add function. Patch by Mark Dickinson and Victor Stinner. + +.. + +.. date: 2024-03-11-17-04-55 +.. gh-issue: 116608 +.. nonce: 30f58- +.. section: Library + +The :mod:`importlib.resources` functions +:func:`~importlib.resources.is_resource()`, +:func:`~importlib.resources.open_binary()`, +:func:`~importlib.resources.open_text()`, +:func:`~importlib.resources.path()`, +:func:`~importlib.resources.read_binary()`, and +:func:`~importlib.resources.read_text()` are un-deprecated, and support +subdirectories via multiple positional arguments. The +:func:`~importlib.resources.contents()` function also allows subdirectories, +but remains deprecated. + +.. + +.. date: 2024-03-08-11-31-49 +.. gh-issue: 116484 +.. nonce: VMAsU7 +.. section: Library + +Change automatically generated :class:`tkinter.Checkbutton` widget names to +avoid collisions with automatically generated +:class:`tkinter.ttk.Checkbutton` widget names within the same parent widget. + +.. + +.. date: 2024-03-07-11-10-27 +.. gh-issue: 114314 +.. nonce: iEhAMH +.. section: Library + +In :mod:`ctypes`, ctype data is now stored in type objects directly rather +than in a dict subclass. This is an internal change that should not affect +usage. + +.. + +.. date: 2024-03-06-18-30-37 +.. gh-issue: 116401 +.. nonce: 3Wcda2 +.. section: Library + +Fix blocking :func:`os.fwalk` and :func:`shutil.rmtree` on opening named +pipe. + +.. + +.. date: 2024-03-05-19-56-29 +.. gh-issue: 71052 +.. nonce: PMDK-- +.. section: Library + +Implement :func:`ctypes.util.find_library` on Android. + +.. + +.. date: 2024-03-01-20-23-57 +.. gh-issue: 90535 +.. nonce: wXm-jC +.. section: Library + +Fix support of *interval* values > 1 in +:class:`logging.TimedRotatingFileHandler` for ``when='MIDNIGHT'`` and +``when='Wx'``. + +.. + +.. date: 2024-02-26-10-06-50 +.. gh-issue: 113308 +.. nonce: MbvOFt +.. section: Library + +Remove some internal protected parts from :mod:`uuid`: +``_has_uuid_generate_time_safe``, ``_netbios_getnode``, +``_ipconfig_getnode``, and ``_load_system_functions``. They were unused. + +.. + +.. date: 2024-02-18-09-50-31 +.. gh-issue: 115627 +.. nonce: HGchj0 +.. section: Library + +Fix the :mod:`ssl` module error handling of connection terminate by peer. It +now throws an OSError with the appropriate error code instead of an +EOFError. + +.. + +.. date: 2024-02-01-08-09-20 +.. gh-issue: 114847 +.. nonce: -JrWrR +.. section: Library + +Speed up :func:`os.path.realpath` on non-Windows platforms. + +.. + +.. date: 2024-02-01-03-09-38 +.. gh-issue: 114271 +.. nonce: raCkt5 +.. section: Library + +Fix a race in ``threading.Thread.join()``. + +``threading._MainThread`` now always represents the main thread of the main +interpreter. + +``PyThreadState.on_delete`` and ``PyThreadState.on_delete_data`` have been +removed. + +.. + +.. date: 2024-01-22-15-50-58 +.. gh-issue: 113538 +.. nonce: v2wrwg +.. section: Library + +Add :meth:`asyncio.Server.close_clients` and +:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully +close an asyncio server. + +.. + +.. date: 2024-01-02-22-47-12 +.. gh-issue: 85287 +.. nonce: ZC5DLj +.. section: Library + +Changes Unicode codecs to return UnicodeEncodeError or UnicodeDecodeError, +rather than just UnicodeError. + +.. + +.. date: 2023-12-28-22-52-45 +.. gh-issue: 113548 +.. nonce: j6TJ7O +.. section: Library + +:mod:`pdb` now allows CLI arguments to ``pdb -m``. + +.. + +.. date: 2023-12-11-00-51-51 +.. gh-issue: 112948 +.. nonce: k-OKp5 +.. section: Library + +Make completion of :mod:`pdb` similar to Python REPL + +.. + +.. date: 2023-06-16-19-17-06 +.. gh-issue: 105866 +.. nonce: 0NBveV +.. section: Library + +Fixed ``_get_slots`` bug which caused error when defining dataclasses with +slots and a weakref_slot. + +.. + +.. date: 2023-05-06-05-00-42 +.. gh-issue: 96471 +.. nonce: S3X5I- +.. section: Library + +Add :py:class:`asyncio.Queue` termination with +:py:meth:`~asyncio.Queue.shutdown` method. + +.. + +.. date: 2022-06-22-14-45-32 +.. gh-issue: 89739 +.. nonce: CqZcRL +.. section: Library + +The :mod:`zipimport` module can now read ZIP64 files. + +.. + +.. bpo: 33533 +.. date: 2020-10-02-17-35-19 +.. nonce: GLIhM5 +.. section: Library + +:func:`asyncio.as_completed` now returns an object that is both an +asynchronous iterator and plain iterator. The new asynchronous iteration +pattern allows for easier correlation between prior tasks and their +completed results. This is a closer match to +:func:`concurrent.futures.as_completed`'s iteration pattern. Patch by Justin +Arthur. + +.. + +.. bpo: 27578 +.. date: 2020-06-11-16-20-33 +.. nonce: CIA-fu +.. section: Library + +:func:`inspect.getsource` (and related functions) work with empty module +files, returning ``'\n'`` (or reasonable equivalent) instead of raising +``OSError``. Patch by Kernc. + +.. + +.. bpo: 37141 +.. date: 2019-09-26-17-52-52 +.. nonce: onYY2- +.. section: Library + +Accept an iterable of separators in :meth:`asyncio.StreamReader.readuntil`, +stopping when one of them is encountered. + +.. + +.. date: 2019-08-27-01-03-26 +.. gh-issue: 66543 +.. nonce: _TRpYr +.. section: Library + +Make :func:`mimetypes.guess_type` properly parsing of URLs with only a host +name, URLs containing fragment or query, and filenames with only a UNC +sharepoint on Windows. Based on patch by Dong-hee Na. + +.. + +.. bpo: 15010 +.. date: 2019-08-12-19-08-06 +.. nonce: 3bY2CF +.. section: Library + +:meth:`unittest.TestLoader.discover` now saves the original value of +``unittest.TestLoader._top_level_dir`` and restores it at the end of the +call. + +.. + +.. date: 2024-03-20-15-12-37 +.. gh-issue: 115977 +.. nonce: IMLi6K +.. section: Documentation + +Remove compatibilty references to Emscripten. + +.. + +.. date: 2024-03-20-12-41-47 +.. gh-issue: 114099 +.. nonce: ad_Ck9 +.. section: Documentation + +Add an iOS platform guide, and flag modules not available on iOS. + +.. + +.. date: 2022-04-15-13-15-23 +.. gh-issue: 91565 +.. nonce: OznXwC +.. section: Documentation + +Changes to documentation files and config outputs to reflect the new +location for reporting bugs - i.e. GitHub rather than bugs.python.org. + +.. + +.. date: 2024-03-25-21-31-49 +.. gh-issue: 83434 +.. nonce: U7Z8cY +.. section: Tests + +Disable JUnit XML output (``--junit-xml=FILE`` command line option) in +regrtest when hunting for reference leaks (``-R`` option). Patch by Victor +Stinner. + +.. + +.. date: 2024-03-24-23-49-25 +.. gh-issue: 117187 +.. nonce: eMLT5n +.. section: Tests + +Fix XML tests for vanilla Expat <2.6.0. + +.. + +.. date: 2024-03-21-11-32-29 +.. gh-issue: 116333 +.. nonce: F-9Ram +.. section: Tests + +Tests of TLS related things (error codes, etc) were updated to be more +lenient about specific error message strings and behaviors as seen in the +BoringSSL and AWS-LC forks of OpenSSL. + +.. + +.. date: 2024-03-20-14-19-32 +.. gh-issue: 117089 +.. nonce: WwR1Z1 +.. section: Tests + +Consolidated tests for importlib.metadata in their own ``metadata`` package. + +.. + +.. date: 2024-03-13-12-06-49 +.. gh-issue: 115979 +.. nonce: zsNpQD +.. section: Tests + +Update test_importlib so that it passes under WASI SDK 21. + +.. + +.. date: 2024-03-11-23-20-28 +.. gh-issue: 112536 +.. nonce: Qv1RrX +.. section: Tests + +Add --tsan to test.regrtest for running TSAN tests in reasonable execution +times. Patch by Donghee Na. + +.. + +.. date: 2024-03-06-11-00-36 +.. gh-issue: 116307 +.. nonce: Uij0t_ +.. section: Tests + +Added import helper ``isolated_modules`` as ``CleanImport`` does not remove +modules imported during the context. Use it in importlib.resources tests to +avoid leaving ``mod`` around to impede importlib.metadata tests. + +.. + +.. date: 2024-03-13-16-16-43 +.. gh-issue: 114736 +.. nonce: ZhmauG +.. section: Build + +Have WASI builds use WASI SDK 21. + +.. + +.. date: 2024-03-08-17-05-15 +.. gh-issue: 115983 +.. nonce: ZQqk0Q +.. section: Build + +Skip building test modules that must be built as shared under WASI. + +.. + +.. date: 2024-03-06-17-26-55 +.. gh-issue: 71052 +.. nonce: vLbu9u +.. section: Build + +Add Android build script and instructions. + +.. + +.. date: 2024-03-28-22-12-00 +.. gh-issue: 117267 +.. nonce: K_tki1 +.. section: Windows + +Ensure ``DirEntry.stat().st_ctime`` behaves consistently with +:func:`os.stat` during the deprecation period of ``st_ctime`` by containing +the same value as ``st_birthtime``. After the deprecation period, +``st_ctime`` will be the metadata change time (or unavailable through +``DirEntry``), and only ``st_birthtime`` will contain the creation time. + +.. + +.. date: 2024-03-14-20-46-23 +.. gh-issue: 116195 +.. nonce: Cu_rYs +.. section: Windows + +Improves performance of :func:`os.getppid` by using an alternate system API +when available. Contributed by vxiiduu. + +.. + +.. date: 2024-03-14-09-14-21 +.. gh-issue: 88494 +.. nonce: Bwfmp7 +.. section: Windows + +On Windows, :func:`time.monotonic()` now uses the +``QueryPerformanceCounter()`` clock to have a resolution better than 1 us, +instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. +Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-01-58-22 +.. gh-issue: 116773 +.. nonce: H2UldY +.. section: Windows + +Fix instances of ``<_overlapped.Overlapped object at 0xXXX> still has +pending operation at deallocation, the process may crash``. + +.. + +.. date: 2024-02-24-23-03-43 +.. gh-issue: 91227 +.. nonce: sL4zWC +.. section: Windows + +Fix the asyncio ProactorEventLoop implementation so that sending a datagram +to an address that is not listening does not prevent receiving any more +datagrams. + +.. + +.. date: 2024-02-08-14-48-15 +.. gh-issue: 115119 +.. nonce: qMt32O +.. section: Windows + +Switched from vendored ``libmpdecimal`` code to a separately-hosted external +package in the ``cpython-source-deps`` repository when building the +``_decimal`` module. + +.. + +.. date: 2024-04-08-18-53-33 +.. gh-issue: 117642 +.. nonce: _-tYH_ +.. section: C API + +Fix :pep:`737` implementation for ``%#T`` and ``%#N``. + +.. + +.. date: 2024-03-22-19-29-24 +.. gh-issue: 87193 +.. nonce: u7O-jY +.. section: C API + +:c:func:`_PyBytes_Resize` can now be called for bytes objects with reference +count > 1, including 1-byte bytes objects. It creates a new bytes object and +destroys the old one if it has reference count > 1. + +.. + +.. date: 2024-03-20-13-13-22 +.. gh-issue: 117021 +.. nonce: 0Q5jBx +.. section: C API + +Fix integer overflow in :c:func:`PyLong_AsPid` on non-Windows 64-bit +platforms. + +.. + +.. date: 2024-03-19-09-49-04 +.. gh-issue: 115756 +.. nonce: 4Ls_Tl +.. section: C API + +:c:func:`!PyCode_GetFirstFree` is an ustable API now and has been renamed to +:c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in +:gh:`115781`) + +.. + +.. date: 2024-03-18-10-58-47 +.. gh-issue: 116869 +.. nonce: lN0GBl +.. section: C API + +Add ``test_cext`` test: build a C extension to check if the Python C API +emits C compiler warnings. Patch by Victor Stinner. + +.. + +.. date: 2024-03-18-09-58-46 +.. gh-issue: 116869 +.. nonce: LFDVKM +.. section: C API + +Make the C API compatible with ``-Werror=declaration-after-statement`` +compiler flag again. Patch by Victor Stinner. + +.. + +.. date: 2024-03-17-22-42-21 +.. gh-issue: 116936 +.. nonce: tNrzfm +.. section: C API + +Add :c:func:`PyType_GetModuleByDef` to the limited C API. Patch by Victor +Stinner. + +.. + +.. date: 2024-03-16-12-21-00 +.. gh-issue: 116809 +.. nonce: JL786L +.. section: C API + +Restore removed private ``_PyErr_ChainExceptions1()`` function. Patch by +Victor Stinner. + +.. + +.. date: 2024-03-15-23-57-33 +.. gh-issue: 115754 +.. nonce: zLdv82 +.. section: C API + +In the limited C API version 3.13, getting ``Py_None``, ``Py_False``, +``Py_True``, ``Py_Ellipsis`` and ``Py_NotImplemented`` singletons is now +implemented as function calls at the stable ABI level to hide implementation +details. Getting these constants still return borrowed references. Patch by +Victor Stinner. + +.. + +.. date: 2024-03-15-23-55-24 +.. gh-issue: 115754 +.. nonce: xnzc__ +.. section: C API + +Add :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` functions +to get constants. For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns +a :term:`strong reference` to the constant zero. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-22-30-07 +.. gh-issue: 111696 +.. nonce: 76UMKi +.. section: C API + +Add support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats to +:c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object +type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for +more information. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-18-00-32 +.. gh-issue: 111696 +.. nonce: L6oIPq +.. section: C API + +Add :c:func:`PyType_GetModuleName` function to get the type's module name. +Equivalent to getting the ``type.__module__`` attribute. Patch by Eric Snow +and Victor Stinner. + +.. + +.. date: 2024-03-14-15-17-11 +.. gh-issue: 111696 +.. nonce: YmnvAi +.. section: C API + +Add :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully +qualified name. Equivalent to ``f"{type.__module__}.{type.__qualname__}"``, +or ``type.__qualname__`` if ``type.__module__`` is not a string or is equal +to ``"builtins"``. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-10-33-58 +.. gh-issue: 85283 +.. nonce: LOgmdU +.. section: C API + +The ``fcntl``, ``grp``, ``pwd``, ``termios``, ``_statistics`` and +``_testconsole`` C extensions are now built with the :ref:`limited C API +`. Patch by Victor Stinner. + +.. + +.. date: 2024-02-28-15-50-01 +.. gh-issue: 111140 +.. nonce: mpwcUg +.. section: C API + +Add additional flags to :c:func:`PyLong_AsNativeBytes` and +:c:func:`PyLong_FromNativeBytes` to allow the caller to determine how to +handle edge cases around values that fill the entire buffer. + +.. + +.. date: 2023-12-12-19-48-31 +.. gh-issue: 113024 +.. nonce: rXcQs7 +.. section: C API + +Add :c:func:`PyObject_GenericHash` function. diff --git a/Misc/NEWS.d/next/Build/2024-03-06-17-26-55.gh-issue-71052.vLbu9u.rst b/Misc/NEWS.d/next/Build/2024-03-06-17-26-55.gh-issue-71052.vLbu9u.rst deleted file mode 100644 index 53776c0216f553..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-03-06-17-26-55.gh-issue-71052.vLbu9u.rst +++ /dev/null @@ -1 +0,0 @@ -Add Android build script and instructions. diff --git a/Misc/NEWS.d/next/Build/2024-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst b/Misc/NEWS.d/next/Build/2024-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst deleted file mode 100644 index a8d39921d59092..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-03-08-17-05-15.gh-issue-115983.ZQqk0Q.rst +++ /dev/null @@ -1 +0,0 @@ -Skip building test modules that must be built as shared under WASI. diff --git a/Misc/NEWS.d/next/Build/2024-03-13-16-16-43.gh-issue-114736.ZhmauG.rst b/Misc/NEWS.d/next/Build/2024-03-13-16-16-43.gh-issue-114736.ZhmauG.rst deleted file mode 100644 index cc863c3a3ceb48..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-03-13-16-16-43.gh-issue-114736.ZhmauG.rst +++ /dev/null @@ -1 +0,0 @@ -Have WASI builds use WASI SDK 21. diff --git a/Misc/NEWS.d/next/C API/2023-12-12-19-48-31.gh-issue-113024.rXcQs7.rst b/Misc/NEWS.d/next/C API/2023-12-12-19-48-31.gh-issue-113024.rXcQs7.rst deleted file mode 100644 index 60ed6e64c3b6b8..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-12-12-19-48-31.gh-issue-113024.rXcQs7.rst +++ /dev/null @@ -1 +0,0 @@ -Add :c:func:`PyObject_GenericHash` function. diff --git a/Misc/NEWS.d/next/C API/2024-02-28-15-50-01.gh-issue-111140.mpwcUg.rst b/Misc/NEWS.d/next/C API/2024-02-28-15-50-01.gh-issue-111140.mpwcUg.rst deleted file mode 100644 index 113db93d186009..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-02-28-15-50-01.gh-issue-111140.mpwcUg.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add additional flags to :c:func:`PyLong_AsNativeBytes` and -:c:func:`PyLong_FromNativeBytes` to allow the caller to determine how to handle -edge cases around values that fill the entire buffer. diff --git a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst deleted file mode 100644 index c8e6b1b1e6ed62..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst +++ /dev/null @@ -1,3 +0,0 @@ -The ``fcntl``, ``grp``, ``pwd``, ``termios``, ``_statistics`` and -``_testconsole`` C extensions are now built with the :ref:`limited C API -`. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-14-15-17-11.gh-issue-111696.YmnvAi.rst b/Misc/NEWS.d/next/C API/2024-03-14-15-17-11.gh-issue-111696.YmnvAi.rst deleted file mode 100644 index 3d87c56bf2493a..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-14-15-17-11.gh-issue-111696.YmnvAi.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully -qualified name. Equivalent to ``f"{type.__module__}.{type.__qualname__}"``, or -``type.__qualname__`` if ``type.__module__`` is not a string or is equal to -``"builtins"``. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-14-18-00-32.gh-issue-111696.L6oIPq.rst b/Misc/NEWS.d/next/C API/2024-03-14-18-00-32.gh-issue-111696.L6oIPq.rst deleted file mode 100644 index 7973d7b16e5826..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-14-18-00-32.gh-issue-111696.L6oIPq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :c:func:`PyType_GetModuleName` function to get the type's module name. -Equivalent to getting the ``type.__module__`` attribute. Patch by Eric Snow -and Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-14-22-30-07.gh-issue-111696.76UMKi.rst b/Misc/NEWS.d/next/C API/2024-03-14-22-30-07.gh-issue-111696.76UMKi.rst deleted file mode 100644 index 44c15e4e6a8256..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-14-22-30-07.gh-issue-111696.76UMKi.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats to -:c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object -type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for -more information. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-15-23-55-24.gh-issue-115754.xnzc__.rst b/Misc/NEWS.d/next/C API/2024-03-15-23-55-24.gh-issue-115754.xnzc__.rst deleted file mode 100644 index d76c98ee54056d..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-15-23-55-24.gh-issue-115754.xnzc__.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` functions to -get constants. For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns a -:term:`strong reference` to the constant zero. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-15-23-57-33.gh-issue-115754.zLdv82.rst b/Misc/NEWS.d/next/C API/2024-03-15-23-57-33.gh-issue-115754.zLdv82.rst deleted file mode 100644 index feff0c0897eae1..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-15-23-57-33.gh-issue-115754.zLdv82.rst +++ /dev/null @@ -1,5 +0,0 @@ -In the limited C API version 3.13, getting ``Py_None``, ``Py_False``, -``Py_True``, ``Py_Ellipsis`` and ``Py_NotImplemented`` singletons is now -implemented as function calls at the stable ABI level to hide implementation -details. Getting these constants still return borrowed references. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-16-12-21-00.gh-issue-116809.JL786L.rst b/Misc/NEWS.d/next/C API/2024-03-16-12-21-00.gh-issue-116809.JL786L.rst deleted file mode 100644 index a122e1b45b959a..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-16-12-21-00.gh-issue-116809.JL786L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restore removed private ``_PyErr_ChainExceptions1()`` function. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-17-22-42-21.gh-issue-116936.tNrzfm.rst b/Misc/NEWS.d/next/C API/2024-03-17-22-42-21.gh-issue-116936.tNrzfm.rst deleted file mode 100644 index bd2abc94082a5a..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-17-22-42-21.gh-issue-116936.tNrzfm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :c:func:`PyType_GetModuleByDef` to the limited C API. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst b/Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst deleted file mode 100644 index 9b9d943f2e6d19..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make the C API compatible with ``-Werror=declaration-after-statement`` -compiler flag again. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst b/Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst deleted file mode 100644 index 71044b4930355a..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``test_cext`` test: build a C extension to check if the Python C API -emits C compiler warnings. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst b/Misc/NEWS.d/next/C API/2024-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst deleted file mode 100644 index 6960395fe229a3..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst +++ /dev/null @@ -1,3 +0,0 @@ -:c:func:`!PyCode_GetFirstFree` is an ustable API now and has been renamed to -:c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in -:gh:`115781`) diff --git a/Misc/NEWS.d/next/C API/2024-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst b/Misc/NEWS.d/next/C API/2024-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst deleted file mode 100644 index 2f93e1e6da00aa..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix integer overflow in :c:func:`PyLong_AsPid` on non-Windows 64-bit -platforms. diff --git a/Misc/NEWS.d/next/C API/2024-03-22-19-29-24.gh-issue-87193.u7O-jY.rst b/Misc/NEWS.d/next/C API/2024-03-22-19-29-24.gh-issue-87193.u7O-jY.rst deleted file mode 100644 index cb921a9c7bf36e..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-03-22-19-29-24.gh-issue-87193.u7O-jY.rst +++ /dev/null @@ -1,3 +0,0 @@ -:c:func:`_PyBytes_Resize` can now be called for bytes objects with reference -count > 1, including 1-byte bytes objects. It creates a new bytes object and -destroys the old one if it has reference count > 1. diff --git a/Misc/NEWS.d/next/C API/2024-04-08-18-53-33.gh-issue-117642._-tYH_.rst b/Misc/NEWS.d/next/C API/2024-04-08-18-53-33.gh-issue-117642._-tYH_.rst deleted file mode 100644 index edef2777717014..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-04-08-18-53-33.gh-issue-117642._-tYH_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :pep:`737` implementation for ``%#T`` and ``%#N``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-04-22-33-01.bpo-24612.SsTuUX.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-04-22-33-01.bpo-24612.SsTuUX.rst deleted file mode 100644 index d54ffc4b76db11..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-04-22-33-01.bpo-24612.SsTuUX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve the :exc:`SyntaxError` that happens when 'not' appears after an -operator. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-09-33-48.gh-issue-97901.BOLluU.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-09-33-48.gh-issue-97901.BOLluU.rst deleted file mode 100644 index 4d2bd65ea1fee6..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-09-33-48.gh-issue-97901.BOLluU.rst +++ /dev/null @@ -1 +0,0 @@ -Mime type ``text/rtf`` is now supported by :mod:`mimetypes`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-14-00-05-17.gh-issue-109870.oKpJ3P.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-14-00-05-17.gh-issue-109870.oKpJ3P.rst deleted file mode 100644 index 390bb1260ea843..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-10-14-00-05-17.gh-issue-109870.oKpJ3P.rst +++ /dev/null @@ -1,3 +0,0 @@ -Dataclasses now calls :func:`exec` once per dataclass, instead of once -per method being added. This can speed up dataclass creation by up to -20%. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst deleted file mode 100644 index 893904bcecea8a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst +++ /dev/null @@ -1,12 +0,0 @@ -Implement an incremental cyclic garbage collector. By collecting the old -generation in increments, there is no need for a full heap scan. This can -hugely reduce maximum pause time for programs with large heaps. - -Reduce the number of generations from three to two. The old generation is -split into two spaces, "visited" and "pending". - -Collection happens in two steps:: -* An increment is formed from the young generation and a small part of the pending space. -* This increment is scanned and the survivors moved to the end of the visited space. - -When the collecting space becomes empty, the two spaces are swapped. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst deleted file mode 100644 index 5974b1882acb22..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst +++ /dev/null @@ -1,4 +0,0 @@ -The array of values, the ``PyDictValues`` struct is now embedded in the -object during allocation. This provides better performance in the common -case, and does not degrade as much when the object's ``__dict__`` is -materialized. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst deleted file mode 100644 index 78bef746b67d85..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-25-14-17-25.gh-issue-115775.CNbGbJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Compiler populates the new ``__static_attributes__`` field on a class with -the names of attributes of this class which are accessed through self.X from -any function in its body. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst deleted file mode 100644 index 5405a3bdc36f9e..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added a Loader that can discover extension modules in an iOS-style Frameworks -folder. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst deleted file mode 100644 index 82f92789de0a39..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-00-45-39.gh-issue-116554.gYumG5.rst +++ /dev/null @@ -1 +0,0 @@ -``list.sort()`` now exploits more cases of partial ordering, particularly those with long descending runs with sub-runs of equal values. Those are recognized as single runs now (previously, each block of repeated values caused a new run to be created). diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst deleted file mode 100644 index 5b18d04cca64b5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-11-22-05-56.gh-issue-116626.GsyczB.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure ``INSTRUMENTED_CALL_FUNCTION_EX`` always emits :monitoring-event:`CALL` diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst deleted file mode 100644 index ab370d4aa1baee..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Starting new threads and process creation through :func:`os.fork` are now -only prevented once all non-daemon threads exit. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-13-16-55-25.gh-issue-116735.o3w6y8.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-13-16-55-25.gh-issue-116735.o3w6y8.rst deleted file mode 100644 index ca15d484e345db..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-13-16-55-25.gh-issue-116735.o3w6y8.rst +++ /dev/null @@ -1 +0,0 @@ -For ``INSTRUMENTED_CALL_FUNCTION_EX``, set ``arg0`` to ``sys.monitoring.MISSING`` instead of ``None`` for :monitoring-event:`CALL` event. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-21-09-57-57.gh-issue-117114.Qu-p55.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-21-09-57-57.gh-issue-117114.Qu-p55.rst deleted file mode 100644 index c9c028a8dda0e5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-21-09-57-57.gh-issue-117114.Qu-p55.rst +++ /dev/null @@ -1 +0,0 @@ -Make :func:`os.path.isdevdrive` available on all platforms. For those that do not offer Dev Drives, it will always return ``False``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-21-12-10-11.gh-issue-117108._6jIrB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-21-12-10-11.gh-issue-117108._6jIrB.rst deleted file mode 100644 index 57ad9606b05e05..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-21-12-10-11.gh-issue-117108._6jIrB.rst +++ /dev/null @@ -1,3 +0,0 @@ -The cycle GC now chooses the size of increments based on the total heap -size, instead of the rate of object creation. This ensures that it can keep -up with growing heaps. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-25-12-51-12.gh-issue-117108.tNqDEo.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-25-12-51-12.gh-issue-117108.tNqDEo.rst deleted file mode 100644 index a28c83ee6efe40..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-25-12-51-12.gh-issue-117108.tNqDEo.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change the old space bit of objects in the young generation from 0 to -gcstate->visited, so that any objects created during GC will have the old -bit set correctly if they get moved into the old generation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-25-17-04-54.gh-issue-99108.8bjdO6.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-25-17-04-54.gh-issue-99108.8bjdO6.rst deleted file mode 100644 index 184273b42b7e9d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-25-17-04-54.gh-issue-99108.8bjdO6.rst +++ /dev/null @@ -1,6 +0,0 @@ -Updated the :mod:`hashlib` built-in `HACL\* project`_ C code from upstream -that we use for many implementations when they are not present via OpenSSL -in a given build. This also avoids the rare potential for a C symbol name -one definition rule linking issue. - -.. _HACL\* project: https://github.com/hacl-star/hacl-star diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-26-17-22-38.gh-issue-117266.Kwh79O.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-26-17-22-38.gh-issue-117266.Kwh79O.rst deleted file mode 100644 index 5055954676b9ab..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-26-17-22-38.gh-issue-117266.Kwh79O.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crashes for certain user-created subclasses of :class:`ast.AST`. Such -classes are now expected to set the ``_field_types`` attribute. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-28-19-13-20.gh-issue-117335.d6uKJu.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-28-19-13-20.gh-issue-117335.d6uKJu.rst deleted file mode 100644 index e419b2e97f3886..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-28-19-13-20.gh-issue-117335.d6uKJu.rst +++ /dev/null @@ -1 +0,0 @@ -Raise TypeError for non-sequences for :func:`ntpath.commonpath`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst deleted file mode 100644 index 7a7bc689002017..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-29-15-04-13.gh-issue-117349.OB9kQQ.rst +++ /dev/null @@ -1 +0,0 @@ -Optimise several functions in :mod:`os.path`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-29-21-43-19.gh-issue-117381.fT0JFM.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-29-21-43-19.gh-issue-117381.fT0JFM.rst deleted file mode 100644 index 88b6c32e971e72..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-29-21-43-19.gh-issue-117381.fT0JFM.rst +++ /dev/null @@ -1 +0,0 @@ -Fix error message for :func:`ntpath.commonpath`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-06-16-49.gh-issue-109120.X485oN.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-06-16-49.gh-issue-109120.X485oN.rst deleted file mode 100644 index 32e70b22f778e1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-06-16-49.gh-issue-109120.X485oN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added handle of incorrect star expressions, e.g ``f(3, *)``. Patch by -Grigoryev Semyon diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-10-04-57.gh-issue-117411.YdyVmG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-10-04-57.gh-issue-117411.YdyVmG.rst deleted file mode 100644 index 73c60ee33a5413..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-10-04-57.gh-issue-117411.YdyVmG.rst +++ /dev/null @@ -1 +0,0 @@ -Move ``PyFutureFeatures`` to an internal header and make it private. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst deleted file mode 100644 index 83f243ae214f7d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-02-17-37-35.gh-issue-117431.vDKAOn.rst +++ /dev/null @@ -1,10 +0,0 @@ -Improve the performance of the following :class:`str` methods -by adapting them to the :c:macro:`METH_FASTCALL` calling convention: - -* :meth:`~str.count` -* :meth:`~str.endswith` -* :meth:`~str.find` -* :meth:`~str.index` -* :meth:`~str.rfind` -* :meth:`~str.rindex` -* :meth:`~str.startswith` diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-03-09-49-15.gh-issue-117431.WAqRgc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-03-09-49-15.gh-issue-117431.WAqRgc.rst deleted file mode 100644 index 17374d0d5c575b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-03-09-49-15.gh-issue-117431.WAqRgc.rst +++ /dev/null @@ -1,6 +0,0 @@ -Improve the performance of the following :class:`bytes` and -:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` -calling convention: - -* :meth:`!endswith` -* :meth:`!startswith` diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-03-13-44-04.gh-issue-116968.zgcdG2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-03-13-44-04.gh-issue-116968.zgcdG2.rst deleted file mode 100644 index dc5beee0022181..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-03-13-44-04.gh-issue-116968.zgcdG2.rst +++ /dev/null @@ -1,11 +0,0 @@ -Introduce a unified 16-bit backoff counter type (``_Py_BackoffCounter``), -shared between the Tier 1 adaptive specializer and the Tier 2 optimizer. The -API used for adaptive specialization counters is changed but the behavior is -(supposed to be) identical. - -The behavior of the Tier 2 counters is changed: - -* There are no longer dynamic thresholds (we never varied these). -* All counters now use the same exponential backoff. -* The counter for ``JUMP_BACKWARD`` starts counting down from 16. -* The ``temperature`` in side exits starts counting down from 64. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-04-13-42-59.gh-issue-117494.GPQH64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-04-13-42-59.gh-issue-117494.GPQH64.rst deleted file mode 100644 index 3b550eda64834b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-04-13-42-59.gh-issue-117494.GPQH64.rst +++ /dev/null @@ -1 +0,0 @@ -Refactored the instruction sequence data structure out of compile.c into instruction_sequence.c. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst deleted file mode 100644 index fd6a6097a89154..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-06-16-42-34.gh-issue-117584.hqk9Hn.rst +++ /dev/null @@ -1 +0,0 @@ -Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath()`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-08-20-26-15.gh-issue-117648.NzVEa7.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-08-20-26-15.gh-issue-117648.NzVEa7.rst deleted file mode 100644 index c7e0dfcc461fc9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-08-20-26-15.gh-issue-117648.NzVEa7.rst +++ /dev/null @@ -1 +0,0 @@ -Speedup :func:`os.path.join` by up to 6% on Windows. diff --git a/Misc/NEWS.d/next/Documentation/2022-04-15-13-15-23.gh-issue-91565.OznXwC.rst b/Misc/NEWS.d/next/Documentation/2022-04-15-13-15-23.gh-issue-91565.OznXwC.rst deleted file mode 100644 index df97e2c447ef58..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2022-04-15-13-15-23.gh-issue-91565.OznXwC.rst +++ /dev/null @@ -1 +0,0 @@ -Changes to documentation files and config outputs to reflect the new location for reporting bugs - i.e. GitHub rather than bugs.python.org. diff --git a/Misc/NEWS.d/next/Documentation/2024-03-20-12-41-47.gh-issue-114099.ad_Ck9.rst b/Misc/NEWS.d/next/Documentation/2024-03-20-12-41-47.gh-issue-114099.ad_Ck9.rst deleted file mode 100644 index c6f403ee899162..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2024-03-20-12-41-47.gh-issue-114099.ad_Ck9.rst +++ /dev/null @@ -1 +0,0 @@ -Add an iOS platform guide, and flag modules not available on iOS. diff --git a/Misc/NEWS.d/next/Documentation/2024-03-20-15-12-37.gh-issue-115977.IMLi6K.rst b/Misc/NEWS.d/next/Documentation/2024-03-20-15-12-37.gh-issue-115977.IMLi6K.rst deleted file mode 100644 index 5f04e93d9a862b..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2024-03-20-15-12-37.gh-issue-115977.IMLi6K.rst +++ /dev/null @@ -1 +0,0 @@ -Remove compatibilty references to Emscripten. diff --git a/Misc/NEWS.d/next/Library/2019-08-12-19-08-06.bpo-15010.3bY2CF.rst b/Misc/NEWS.d/next/Library/2019-08-12-19-08-06.bpo-15010.3bY2CF.rst deleted file mode 100644 index f61a45ed98abad..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-08-12-19-08-06.bpo-15010.3bY2CF.rst +++ /dev/null @@ -1,3 +0,0 @@ -:meth:`unittest.TestLoader.discover` now saves the original value of -``unittest.TestLoader._top_level_dir`` and restores it at the end of the -call. diff --git a/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.gh-issue-66543._TRpYr.rst b/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.gh-issue-66543._TRpYr.rst deleted file mode 100644 index 62f7aa2490bb73..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.gh-issue-66543._TRpYr.rst +++ /dev/null @@ -1,4 +0,0 @@ -Make :func:`mimetypes.guess_type` properly parsing of URLs with only a host -name, URLs containing fragment or query, and filenames with only a UNC -sharepoint on Windows. -Based on patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2019-09-26-17-52-52.bpo-37141.onYY2-.rst b/Misc/NEWS.d/next/Library/2019-09-26-17-52-52.bpo-37141.onYY2-.rst deleted file mode 100644 index d916f319947dc4..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-09-26-17-52-52.bpo-37141.onYY2-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Accept an iterable of separators in :meth:`asyncio.StreamReader.readuntil`, stopping -when one of them is encountered. diff --git a/Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst b/Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst deleted file mode 100644 index df58a7ede45521..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`inspect.getsource` (and related functions) work with -empty module files, returning ``'\n'`` (or reasonable equivalent) -instead of raising ``OSError``. Patch by Kernc. diff --git a/Misc/NEWS.d/next/Library/2020-10-02-17-35-19.bpo-33533.GLIhM5.rst b/Misc/NEWS.d/next/Library/2020-10-02-17-35-19.bpo-33533.GLIhM5.rst deleted file mode 100644 index 3ffd723cf1082a..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-10-02-17-35-19.bpo-33533.GLIhM5.rst +++ /dev/null @@ -1,5 +0,0 @@ -:func:`asyncio.as_completed` now returns an object that is both an asynchronous -iterator and plain iterator. The new asynchronous iteration pattern allows for -easier correlation between prior tasks and their completed results. This is -a closer match to :func:`concurrent.futures.as_completed`'s iteration pattern. -Patch by Justin Arthur. diff --git a/Misc/NEWS.d/next/Library/2022-06-22-14-45-32.gh-issue-89739.CqZcRL.rst b/Misc/NEWS.d/next/Library/2022-06-22-14-45-32.gh-issue-89739.CqZcRL.rst deleted file mode 100644 index 0358c0107cb697..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-06-22-14-45-32.gh-issue-89739.CqZcRL.rst +++ /dev/null @@ -1 +0,0 @@ -The :mod:`zipimport` module can now read ZIP64 files. diff --git a/Misc/NEWS.d/next/Library/2023-05-06-05-00-42.gh-issue-96471.S3X5I-.rst b/Misc/NEWS.d/next/Library/2023-05-06-05-00-42.gh-issue-96471.S3X5I-.rst deleted file mode 100644 index 128a85d3d73ddf..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-06-05-00-42.gh-issue-96471.S3X5I-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :py:class:`asyncio.Queue` termination with -:py:meth:`~asyncio.Queue.shutdown` method. diff --git a/Misc/NEWS.d/next/Library/2023-06-16-19-17-06.gh-issue-105866.0NBveV.rst b/Misc/NEWS.d/next/Library/2023-06-16-19-17-06.gh-issue-105866.0NBveV.rst deleted file mode 100644 index 28eae1232742f7..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-16-19-17-06.gh-issue-105866.0NBveV.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed ``_get_slots`` bug which caused error when defining dataclasses with slots and a weakref_slot. diff --git a/Misc/NEWS.d/next/Library/2023-12-11-00-51-51.gh-issue-112948.k-OKp5.rst b/Misc/NEWS.d/next/Library/2023-12-11-00-51-51.gh-issue-112948.k-OKp5.rst deleted file mode 100644 index 0925a7caba6f07..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-11-00-51-51.gh-issue-112948.k-OKp5.rst +++ /dev/null @@ -1 +0,0 @@ -Make completion of :mod:`pdb` similar to Python REPL diff --git a/Misc/NEWS.d/next/Library/2023-12-28-22-52-45.gh-issue-113548.j6TJ7O.rst b/Misc/NEWS.d/next/Library/2023-12-28-22-52-45.gh-issue-113548.j6TJ7O.rst deleted file mode 100644 index 972ddeb54822e2..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-28-22-52-45.gh-issue-113548.j6TJ7O.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`pdb` now allows CLI arguments to ``pdb -m``. diff --git a/Misc/NEWS.d/next/Library/2024-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst b/Misc/NEWS.d/next/Library/2024-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst deleted file mode 100644 index e6d031fbc93e83..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-02-22-47-12.gh-issue-85287.ZC5DLj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Changes Unicode codecs to return UnicodeEncodeError or UnicodeDecodeError, -rather than just UnicodeError. diff --git a/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst deleted file mode 100644 index 5c59af98e136bb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :meth:`asyncio.Server.close_clients` and -:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully -close an asyncio server. diff --git a/Misc/NEWS.d/next/Library/2024-02-01-03-09-38.gh-issue-114271.raCkt5.rst b/Misc/NEWS.d/next/Library/2024-02-01-03-09-38.gh-issue-114271.raCkt5.rst deleted file mode 100644 index 2cd35eeda830b9..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-01-03-09-38.gh-issue-114271.raCkt5.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fix a race in ``threading.Thread.join()``. - -``threading._MainThread`` now always represents the main thread of the main -interpreter. - -``PyThreadState.on_delete`` and ``PyThreadState.on_delete_data`` have been -removed. diff --git a/Misc/NEWS.d/next/Library/2024-02-01-08-09-20.gh-issue-114847.-JrWrR.rst b/Misc/NEWS.d/next/Library/2024-02-01-08-09-20.gh-issue-114847.-JrWrR.rst deleted file mode 100644 index bf011fed3efdbc..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-01-08-09-20.gh-issue-114847.-JrWrR.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :func:`os.path.realpath` on non-Windows platforms. diff --git a/Misc/NEWS.d/next/Library/2024-02-18-09-50-31.gh-issue-115627.HGchj0.rst b/Misc/NEWS.d/next/Library/2024-02-18-09-50-31.gh-issue-115627.HGchj0.rst deleted file mode 100644 index 75d926ab59d557..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-18-09-50-31.gh-issue-115627.HGchj0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the :mod:`ssl` module error handling of connection terminate by peer. -It now throws an OSError with the appropriate error code instead of an EOFError. diff --git a/Misc/NEWS.d/next/Library/2024-02-26-10-06-50.gh-issue-113308.MbvOFt.rst b/Misc/NEWS.d/next/Library/2024-02-26-10-06-50.gh-issue-113308.MbvOFt.rst deleted file mode 100644 index c4c242fe3d578f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-26-10-06-50.gh-issue-113308.MbvOFt.rst +++ /dev/null @@ -1,4 +0,0 @@ -Remove some internal protected parts from :mod:`uuid`: -``_has_uuid_generate_time_safe``, ``_netbios_getnode``, -``_ipconfig_getnode``, and ``_load_system_functions``. -They were unused. diff --git a/Misc/NEWS.d/next/Library/2024-03-01-20-23-57.gh-issue-90535.wXm-jC.rst b/Misc/NEWS.d/next/Library/2024-03-01-20-23-57.gh-issue-90535.wXm-jC.rst deleted file mode 100644 index 9af4efabb6b5b2..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-01-20-23-57.gh-issue-90535.wXm-jC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix support of *interval* values > 1 in -:class:`logging.TimedRotatingFileHandler` for ``when='MIDNIGHT'`` and -``when='Wx'``. diff --git a/Misc/NEWS.d/next/Library/2024-03-05-19-56-29.gh-issue-71052.PMDK--.rst b/Misc/NEWS.d/next/Library/2024-03-05-19-56-29.gh-issue-71052.PMDK--.rst deleted file mode 100644 index ddca54c7c9ed7b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-05-19-56-29.gh-issue-71052.PMDK--.rst +++ /dev/null @@ -1 +0,0 @@ -Implement :func:`ctypes.util.find_library` on Android. diff --git a/Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst b/Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst deleted file mode 100644 index 121f0065ecca95..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-06-18-30-37.gh-issue-116401.3Wcda2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix blocking :func:`os.fwalk` and :func:`shutil.rmtree` on opening named -pipe. diff --git a/Misc/NEWS.d/next/Library/2024-03-07-11-10-27.gh-issue-114314.iEhAMH.rst b/Misc/NEWS.d/next/Library/2024-03-07-11-10-27.gh-issue-114314.iEhAMH.rst deleted file mode 100644 index c241d966f9087d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-07-11-10-27.gh-issue-114314.iEhAMH.rst +++ /dev/null @@ -1,3 +0,0 @@ -In :mod:`ctypes`, ctype data is now stored in type objects directly rather -than in a dict subclass. This is an internal change that should not affect -usage. diff --git a/Misc/NEWS.d/next/Library/2024-03-08-11-31-49.gh-issue-116484.VMAsU7.rst b/Misc/NEWS.d/next/Library/2024-03-08-11-31-49.gh-issue-116484.VMAsU7.rst deleted file mode 100644 index 265c3810466d39..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-08-11-31-49.gh-issue-116484.VMAsU7.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change automatically generated :class:`tkinter.Checkbutton` widget names to -avoid collisions with automatically generated -:class:`tkinter.ttk.Checkbutton` widget names within the same parent widget. diff --git a/Misc/NEWS.d/next/Library/2024-03-11-17-04-55.gh-issue-116608.30f58-.rst b/Misc/NEWS.d/next/Library/2024-03-11-17-04-55.gh-issue-116608.30f58-.rst deleted file mode 100644 index d1536bc47c3ee0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-11-17-04-55.gh-issue-116608.30f58-.rst +++ /dev/null @@ -1,10 +0,0 @@ -The :mod:`importlib.resources` functions -:func:`~importlib.resources.is_resource()`, -:func:`~importlib.resources.open_binary()`, -:func:`~importlib.resources.open_text()`, -:func:`~importlib.resources.path()`, -:func:`~importlib.resources.read_binary()`, and -:func:`~importlib.resources.read_text()` are un-deprecated, and support -subdirectories via multiple positional arguments. -The :func:`~importlib.resources.contents()` function also allows subdirectories, -but remains deprecated. diff --git a/Misc/NEWS.d/next/Library/2024-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst b/Misc/NEWS.d/next/Library/2024-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst deleted file mode 100644 index c91f4eb97e06bc..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-12-17-53-14.gh-issue-73468.z4ZzvJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added new :func:`math.fma` function, wrapping C99's ``fma()`` operation: -fused multiply-add function. Patch by Mark Dickinson and Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-03-12-19-32-17.gh-issue-71042.oI0Ron.rst b/Misc/NEWS.d/next/Library/2024-03-12-19-32-17.gh-issue-71042.oI0Ron.rst deleted file mode 100644 index 3641cbb9b2fc1a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-12-19-32-17.gh-issue-71042.oI0Ron.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :func:`platform.android_ver`, which provides device and OS information -on Android. diff --git a/Misc/NEWS.d/next/Library/2024-03-13-15-45-54.gh-issue-63283.OToJnG.rst b/Misc/NEWS.d/next/Library/2024-03-13-15-45-54.gh-issue-63283.OToJnG.rst deleted file mode 100644 index bb4c3a4a8d741b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-13-15-45-54.gh-issue-63283.OToJnG.rst +++ /dev/null @@ -1,2 +0,0 @@ -In :mod:`encodings.idna`, any capitalization of the the ACE prefix -(``xn--``) is now acceptable. Patch by Pepijn de Vos and Zackery Spytz. diff --git a/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst deleted file mode 100644 index f9a72473be4e2c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst +++ /dev/null @@ -1,9 +0,0 @@ -Fixed various false positives and false negatives in - -* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details) -* :attr:`ipaddress.IPv4Address.is_global` -* :attr:`ipaddress.IPv6Address.is_private` -* :attr:`ipaddress.IPv6Address.is_global` - -Also in the corresponding :class:`ipaddress.IPv4Network` and :class:`ipaddress.IPv6Network` -attributes. diff --git a/Misc/NEWS.d/next/Library/2024-03-14-09-38-51.gh-issue-116647.h0d_zj.rst b/Misc/NEWS.d/next/Library/2024-03-14-09-38-51.gh-issue-116647.h0d_zj.rst deleted file mode 100644 index 081f36bff91633..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-14-09-38-51.gh-issue-116647.h0d_zj.rst +++ /dev/null @@ -1 +0,0 @@ -Fix recursive child in dataclasses diff --git a/Misc/NEWS.d/next/Library/2024-03-14-10-01-23.gh-issue-116811._h5iKP.rst b/Misc/NEWS.d/next/Library/2024-03-14-10-01-23.gh-issue-116811._h5iKP.rst deleted file mode 100644 index 00168632429996..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-14-10-01-23.gh-issue-116811._h5iKP.rst +++ /dev/null @@ -1,2 +0,0 @@ -In ``PathFinder.invalidate_caches``, delegate to -``MetadataPathFinder.invalidate_caches``. diff --git a/Misc/NEWS.d/next/Library/2024-03-14-14-01-46.gh-issue-116764.moB3Lc.rst b/Misc/NEWS.d/next/Library/2024-03-14-14-01-46.gh-issue-116764.moB3Lc.rst deleted file mode 100644 index e92034b0e8b157..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-14-14-01-46.gh-issue-116764.moB3Lc.rst +++ /dev/null @@ -1,4 +0,0 @@ -Restore support of ``None`` and other false values in :mod:`urllib.parse` -functions :func:`~urllib.parse.parse_qs` and -:func:`~urllib.parse.parse_qsl`. Also, they now raise a TypeError for -non-zero integers and non-empty sequences. diff --git a/Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst b/Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst deleted file mode 100644 index 1f77555d5e7d31..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst +++ /dev/null @@ -1,4 +0,0 @@ -On Windows, :func:`time.time()` now uses the -``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better than 1 -us, instead of the ``GetSystemTimeAsFileTime()`` clock which has a resolution -of 15.6 ms. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-03-14-17-24-59.gh-issue-106531.9ehywi.rst b/Misc/NEWS.d/next/Library/2024-03-14-17-24-59.gh-issue-106531.9ehywi.rst deleted file mode 100644 index e2720d333783c0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-14-17-24-59.gh-issue-106531.9ehywi.rst +++ /dev/null @@ -1,5 +0,0 @@ -Refreshed zipfile._path from `zipp 3.18 -`_, providing -better compatibility for PyPy, better glob performance for deeply nested -zipfiles, and providing internal access to ``CompleteDirs.inject`` for use -in other tests (like importlib.resources). diff --git a/Misc/NEWS.d/next/Library/2024-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst b/Misc/NEWS.d/next/Library/2024-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst deleted file mode 100644 index b7024c74f7aa7d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-14-20-59-28.gh-issue-90095.7UaJ1U.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore empty lines and comments in ``.pdbrc`` diff --git a/Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst b/Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst deleted file mode 100644 index fda2ebf7593ed5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst +++ /dev/null @@ -1,2 +0,0 @@ -:class:`_io.WindowsConsoleIO` now emit a warning if a boolean value is -passed as a filedescriptor argument. diff --git a/Misc/NEWS.d/next/Library/2024-03-18-14-36-50.gh-issue-116957.dTCs4f.rst b/Misc/NEWS.d/next/Library/2024-03-18-14-36-50.gh-issue-116957.dTCs4f.rst deleted file mode 100644 index 51fe04957e26bc..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-18-14-36-50.gh-issue-116957.dTCs4f.rst +++ /dev/null @@ -1,3 +0,0 @@ -configparser: Don't leave ConfigParser values in an invalid state (stored as -a list instead of a str) after an earlier read raised DuplicateSectionError -or DuplicateOptionError. diff --git a/Misc/NEWS.d/next/Library/2024-03-19-11-08-26.gh-issue-90872.ghys95.rst b/Misc/NEWS.d/next/Library/2024-03-19-11-08-26.gh-issue-90872.ghys95.rst deleted file mode 100644 index ead68caa9fe88b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-19-11-08-26.gh-issue-90872.ghys95.rst +++ /dev/null @@ -1,3 +0,0 @@ -On Windows, :meth:`subprocess.Popen.wait` no longer calls -``WaitForSingleObject()`` with a negative timeout: pass ``0`` ms if the -timeout is negative. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-03-19-14-35-57.gh-issue-114099.siNSpK.rst b/Misc/NEWS.d/next/Library/2024-03-19-14-35-57.gh-issue-114099.siNSpK.rst deleted file mode 100644 index 9b57cbb812db4a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-19-14-35-57.gh-issue-114099.siNSpK.rst +++ /dev/null @@ -1 +0,0 @@ -Modify standard library to allow for iOS platform differences. diff --git a/Misc/NEWS.d/next/Library/2024-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst b/Misc/NEWS.d/next/Library/2024-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst deleted file mode 100644 index f2da956f66c86b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed :func:`inspect.findsource` for class code objects. diff --git a/Misc/NEWS.d/next/Library/2024-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst b/Misc/NEWS.d/next/Library/2024-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst deleted file mode 100644 index a9094df06037cd..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-20-16-10-29.gh-issue-70647.FpD6Ar.rst +++ /dev/null @@ -1,7 +0,0 @@ -Start the deprecation period for the current behavior of -:func:`datetime.datetime.strptime` and :func:`time.strptime` which always -fails to parse a date string with a :exc:`ValueError` involving a day of -month such as ``strptime("02-29", "%m-%d")`` when a year is **not** -specified and the date happen to be February 29th. This should help avoid -users finding new bugs every four years due to a natural mistaken assumption -about the API when parsing partial date values. diff --git a/Misc/NEWS.d/next/Library/2024-03-20-23-07-58.gh-issue-109653.uu3lrX.rst b/Misc/NEWS.d/next/Library/2024-03-20-23-07-58.gh-issue-109653.uu3lrX.rst deleted file mode 100644 index 38d7634b54c2fe..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-20-23-07-58.gh-issue-109653.uu3lrX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deferred select imports in importlib.metadata and importlib.resources for a -14% speedup. diff --git a/Misc/NEWS.d/next/Library/2024-03-21-07-27-36.gh-issue-117110.9K1InX.rst b/Misc/NEWS.d/next/Library/2024-03-21-07-27-36.gh-issue-117110.9K1InX.rst deleted file mode 100644 index 32f8f81c8d052f..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-21-07-27-36.gh-issue-117110.9K1InX.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug that prevents subclasses of :class:`typing.Any` to be instantiated with arguments. Patch by Chris Fu. diff --git a/Misc/NEWS.d/next/Library/2024-03-21-17-07-38.gh-issue-117084.w1mTpT.rst b/Misc/NEWS.d/next/Library/2024-03-21-17-07-38.gh-issue-117084.w1mTpT.rst deleted file mode 100644 index 6e7790e926b9d2..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-21-17-07-38.gh-issue-117084.w1mTpT.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :mod:`zipfile` extraction for directory entries with the name containing -backslashes on Windows. diff --git a/Misc/NEWS.d/next/Library/2024-03-23-12-28-05.gh-issue-117182.a0KANW.rst b/Misc/NEWS.d/next/Library/2024-03-23-12-28-05.gh-issue-117182.a0KANW.rst deleted file mode 100644 index 6b3b841d9d5d7b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-23-12-28-05.gh-issue-117182.a0KANW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Lazy-loading of modules that modify their own ``__class__`` no longer -reverts the ``__class__`` to :class:`types.ModuleType`. diff --git a/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst b/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst deleted file mode 100644 index 931e615c2b86c5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-23-13-40-13.gh-issue-112383.XuHf3G.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :mod:`dis` module's handling of ``ENTER_EXECUTOR`` instructions. diff --git a/Misc/NEWS.d/next/Library/2024-03-23-14-26-18.gh-issue-117178.vTisTG.rst b/Misc/NEWS.d/next/Library/2024-03-23-14-26-18.gh-issue-117178.vTisTG.rst deleted file mode 100644 index f9c53ebbfc3c96..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-23-14-26-18.gh-issue-117178.vTisTG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix regression in lazy loading of self-referential modules, introduced in -gh-114781. diff --git a/Misc/NEWS.d/next/Library/2024-03-25-00-20-16.gh-issue-117205.yV7xGb.rst b/Misc/NEWS.d/next/Library/2024-03-25-00-20-16.gh-issue-117205.yV7xGb.rst deleted file mode 100644 index 8d8c201afd29fb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-25-00-20-16.gh-issue-117205.yV7xGb.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :func:`compileall.compile_dir` by 20% when using multiprocessing by increasing ``chunksize``. diff --git a/Misc/NEWS.d/next/Library/2024-03-25-21-15-56.gh-issue-117225.oOaZXb.rst b/Misc/NEWS.d/next/Library/2024-03-25-21-15-56.gh-issue-117225.oOaZXb.rst deleted file mode 100644 index b6c4850f608c2a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-25-21-15-56.gh-issue-117225.oOaZXb.rst +++ /dev/null @@ -1,2 +0,0 @@ -doctest: only print "and X failed" when non-zero, don't pluralise "1 items". -Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-03-26-11-48-39.gh-issue-98966.SayV9y.rst b/Misc/NEWS.d/next/Library/2024-03-26-11-48-39.gh-issue-98966.SayV9y.rst deleted file mode 100644 index e819a1e9a0aba0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-26-11-48-39.gh-issue-98966.SayV9y.rst +++ /dev/null @@ -1,2 +0,0 @@ -In :mod:`subprocess`, raise a more informative message when -``stdout=STDOUT``. diff --git a/Misc/NEWS.d/next/Library/2024-03-27-16-43-42.gh-issue-117294.wbXNFv.rst b/Misc/NEWS.d/next/Library/2024-03-27-16-43-42.gh-issue-117294.wbXNFv.rst deleted file mode 100644 index bb351e6399a765..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-27-16-43-42.gh-issue-117294.wbXNFv.rst +++ /dev/null @@ -1,2 +0,0 @@ -A ``DocTestCase`` now reports as skipped if all examples in the doctest are -skipped. diff --git a/Misc/NEWS.d/next/Library/2024-03-27-21-05-52.gh-issue-117310.Bt2wox.rst b/Misc/NEWS.d/next/Library/2024-03-27-21-05-52.gh-issue-117310.Bt2wox.rst deleted file mode 100644 index 429b890b8b609a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-27-21-05-52.gh-issue-117310.Bt2wox.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed an unlikely early & extra ``Py_DECREF`` triggered crash in :mod:`ssl` -when creating a new ``_ssl._SSLContext`` if CPython was built implausibly such -that the default cipher list is empty **or** the SSL library it was linked -against reports a failure from its C ``SSL_CTX_set_cipher_list()`` API. diff --git a/Misc/NEWS.d/next/Library/2024-03-28-13-54-20.gh-issue-88014.zJz31I.rst b/Misc/NEWS.d/next/Library/2024-03-28-13-54-20.gh-issue-88014.zJz31I.rst deleted file mode 100644 index f8bb784e39fbb6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-28-13-54-20.gh-issue-88014.zJz31I.rst +++ /dev/null @@ -1,3 +0,0 @@ -In documentation of :class:`gzip.GzipFile` in module gzip, explain data type -of optional constructor argument *mtime*, and recommend ``mtime = 0`` for -generating deterministic streams. diff --git a/Misc/NEWS.d/next/Library/2024-03-28-17-55-22.gh-issue-66449.4jhuEV.rst b/Misc/NEWS.d/next/Library/2024-03-28-17-55-22.gh-issue-66449.4jhuEV.rst deleted file mode 100644 index 898100b87e1dbd..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-28-17-55-22.gh-issue-66449.4jhuEV.rst +++ /dev/null @@ -1,2 +0,0 @@ -:class:`configparser.ConfigParser` now accepts unnamed sections before named -ones, if configured to do so. diff --git a/Misc/NEWS.d/next/Library/2024-03-29-12-07-26.gh-issue-117348.WjCYvK.rst b/Misc/NEWS.d/next/Library/2024-03-29-12-07-26.gh-issue-117348.WjCYvK.rst deleted file mode 100644 index cd3006c3b7b8f0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-29-12-07-26.gh-issue-117348.WjCYvK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Refactored :meth:`configparser.RawConfigParser._read` to reduce cyclometric -complexity and improve comprehensibility. diff --git a/Misc/NEWS.d/next/Library/2024-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst b/Misc/NEWS.d/next/Library/2024-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst deleted file mode 100644 index 73bd2569c7c9cb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-29-15-58-01.gh-issue-117337.7w3Qwp.rst +++ /dev/null @@ -1,3 +0,0 @@ -Deprecate undocumented :func:`!glob.glob0` and :func:`!glob.glob1` -functions. Use :func:`glob.glob` and pass a directory to its -*root_dir* argument instead. diff --git a/Misc/NEWS.d/next/Library/2024-04-02-13-13-46.gh-issue-117459.jiIZmH.rst b/Misc/NEWS.d/next/Library/2024-04-02-13-13-46.gh-issue-117459.jiIZmH.rst deleted file mode 100644 index 549bd44112befe..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-02-13-13-46.gh-issue-117459.jiIZmH.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`asyncio.asyncio.run_coroutine_threadsafe` now keeps the traceback of :class:`CancelledError`, :class:`TimeoutError` and :class:`InvalidStateError` which are raised in the coroutine. diff --git a/Misc/NEWS.d/next/Library/2024-04-02-20-30-12.gh-issue-114848.YX4pEc.rst b/Misc/NEWS.d/next/Library/2024-04-02-20-30-12.gh-issue-114848.YX4pEc.rst deleted file mode 100644 index 30b1a50976f52d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-02-20-30-12.gh-issue-114848.YX4pEc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Raise :exc:`FileNotFoundError` when ``getcwd()`` returns '(unreachable)', -which can happen on Linux >= 2.6.36 with glibc < 2.27. diff --git a/Misc/NEWS.d/next/Library/2024-04-03-18-36-53.gh-issue-117467.l6rWlj.rst b/Misc/NEWS.d/next/Library/2024-04-03-18-36-53.gh-issue-117467.l6rWlj.rst deleted file mode 100644 index 64ae9ff7b2f0b5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-04-03-18-36-53.gh-issue-117467.l6rWlj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush`. -Patch by Tony Mountifield. diff --git a/Misc/NEWS.d/next/Tests/2024-03-06-11-00-36.gh-issue-116307.Uij0t_.rst b/Misc/NEWS.d/next/Tests/2024-03-06-11-00-36.gh-issue-116307.Uij0t_.rst deleted file mode 100644 index 0bc4be94789f21..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-03-06-11-00-36.gh-issue-116307.Uij0t_.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added import helper ``isolated_modules`` as ``CleanImport`` does not remove -modules imported during the context. Use it in importlib.resources tests to -avoid leaving ``mod`` around to impede importlib.metadata tests. diff --git a/Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst b/Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst deleted file mode 100644 index de9e1c557b093c..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-03-11-23-20-28.gh-issue-112536.Qv1RrX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add --tsan to test.regrtest for running TSAN tests in reasonable execution -times. Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Tests/2024-03-13-12-06-49.gh-issue-115979.zsNpQD.rst b/Misc/NEWS.d/next/Tests/2024-03-13-12-06-49.gh-issue-115979.zsNpQD.rst deleted file mode 100644 index 02bc2b88942e4f..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-03-13-12-06-49.gh-issue-115979.zsNpQD.rst +++ /dev/null @@ -1 +0,0 @@ -Update test_importlib so that it passes under WASI SDK 21. diff --git a/Misc/NEWS.d/next/Tests/2024-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst b/Misc/NEWS.d/next/Tests/2024-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst deleted file mode 100644 index ab0baec8c96035..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst +++ /dev/null @@ -1 +0,0 @@ -Consolidated tests for importlib.metadata in their own ``metadata`` package. diff --git a/Misc/NEWS.d/next/Tests/2024-03-21-11-32-29.gh-issue-116333.F-9Ram.rst b/Misc/NEWS.d/next/Tests/2024-03-21-11-32-29.gh-issue-116333.F-9Ram.rst deleted file mode 100644 index 3fdb6bb3bd7af7..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-03-21-11-32-29.gh-issue-116333.F-9Ram.rst +++ /dev/null @@ -1,3 +0,0 @@ -Tests of TLS related things (error codes, etc) were updated to be more -lenient about specific error message strings and behaviors as seen in the -BoringSSL and AWS-LC forks of OpenSSL. diff --git a/Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst b/Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst deleted file mode 100644 index 0c0b0e0f443396..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst +++ /dev/null @@ -1 +0,0 @@ -Fix XML tests for vanilla Expat <2.6.0. diff --git a/Misc/NEWS.d/next/Tests/2024-03-25-21-31-49.gh-issue-83434.U7Z8cY.rst b/Misc/NEWS.d/next/Tests/2024-03-25-21-31-49.gh-issue-83434.U7Z8cY.rst deleted file mode 100644 index 7b7a8fcf53bb3c..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-03-25-21-31-49.gh-issue-83434.U7Z8cY.rst +++ /dev/null @@ -1,3 +0,0 @@ -Disable JUnit XML output (``--junit-xml=FILE`` command line option) in -regrtest when hunting for reference leaks (``-R`` option). Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst b/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst deleted file mode 100644 index f95fed1084cf4f..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst +++ /dev/null @@ -1,3 +0,0 @@ -Switched from vendored ``libmpdecimal`` code to a separately-hosted external -package in the ``cpython-source-deps`` repository when building the -``_decimal`` module. diff --git a/Misc/NEWS.d/next/Windows/2024-02-24-23-03-43.gh-issue-91227.sL4zWC.rst b/Misc/NEWS.d/next/Windows/2024-02-24-23-03-43.gh-issue-91227.sL4zWC.rst deleted file mode 100644 index 8e53afdd619001..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-24-23-03-43.gh-issue-91227.sL4zWC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix the asyncio ProactorEventLoop implementation so that sending a datagram to an address that is not listening does not prevent receiving any more datagrams. diff --git a/Misc/NEWS.d/next/Windows/2024-03-14-01-58-22.gh-issue-116773.H2UldY.rst b/Misc/NEWS.d/next/Windows/2024-03-14-01-58-22.gh-issue-116773.H2UldY.rst deleted file mode 100644 index 8fc3fe80041d26..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-03-14-01-58-22.gh-issue-116773.H2UldY.rst +++ /dev/null @@ -1 +0,0 @@ -Fix instances of ``<_overlapped.Overlapped object at 0xXXX> still has pending operation at deallocation, the process may crash``. diff --git a/Misc/NEWS.d/next/Windows/2024-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst b/Misc/NEWS.d/next/Windows/2024-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst deleted file mode 100644 index 5a96af0231918f..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-03-14-09-14-21.gh-issue-88494.Bwfmp7.rst +++ /dev/null @@ -1,4 +0,0 @@ -On Windows, :func:`time.monotonic()` now uses the ``QueryPerformanceCounter()`` -clock to have a resolution better than 1 us, instead of the -``GetTickCount64()`` clock which has a resolution of 15.6 ms. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Windows/2024-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst b/Misc/NEWS.d/next/Windows/2024-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst deleted file mode 100644 index 32122d764e870a..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-03-14-20-46-23.gh-issue-116195.Cu_rYs.rst +++ /dev/null @@ -1 +0,0 @@ -Improves performance of :func:`os.getppid` by using an alternate system API when available. Contributed by vxiiduu. diff --git a/Misc/NEWS.d/next/Windows/2024-03-28-22-12-00.gh-issue-117267.K_tki1.rst b/Misc/NEWS.d/next/Windows/2024-03-28-22-12-00.gh-issue-117267.K_tki1.rst deleted file mode 100644 index d3221429850a11..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-03-28-22-12-00.gh-issue-117267.K_tki1.rst +++ /dev/null @@ -1,5 +0,0 @@ -Ensure ``DirEntry.stat().st_ctime`` behaves consistently with -:func:`os.stat` during the deprecation period of ``st_ctime`` by containing -the same value as ``st_birthtime``. After the deprecation period, -``st_ctime`` will be the metadata change time (or unavailable through -``DirEntry``), and only ``st_birthtime`` will contain the creation time. diff --git a/README.rst b/README.rst index 46167b38eab566..cab9519bd7a76c 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.13.0 alpha 5 +This is Python version 3.13.0 alpha 6 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From f2132fcd2a6da7b2b86e80189fa009ce1d2c753b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 9 Apr 2024 06:50:37 -0400 Subject: [PATCH 124/143] gh-117516: Implement typing.TypeIs (#117517) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See PEP 742. Co-authored-by: Carl Meyer Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/typing.rst | 118 ++++++++++++++---- Doc/whatsnew/3.13.rst | 4 + Lib/test/test_typing.py | 56 ++++++++- Lib/typing.py | 96 ++++++++++++-- ...-04-03-16-01-31.gh-issue-117516.7DlHje.rst | 1 + 5 files changed, 236 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-03-16-01-31.gh-issue-117516.7DlHje.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 73214e18d556b2..19dbd376c80d51 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1385,22 +1385,23 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.9 -.. data:: TypeGuard +.. data:: TypeIs - Special typing construct for marking user-defined type guard functions. + Special typing construct for marking user-defined type predicate functions. - ``TypeGuard`` can be used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. - At runtime, functions marked this way should return a boolean. + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and take at + least one positional argument. - ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard":: + conditional expression here is sometimes referred to as a "type predicate":: def is_str(val: str | float): - # "isinstance" type guard + # "isinstance" type predicate if isinstance(val, str): # Type of ``val`` is narrowed to ``str`` ... @@ -1409,8 +1410,73 @@ These can be used as types in annotations. They all support subscription using ... Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeIs[...]`` or + :data:`TypeGuard` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. + + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for a given + function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. + + For example:: + + from typing import assert_type, final, TypeIs + + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass + + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) + + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + If a ``TypeIs`` function is a class or instance method, then the type in + ``TypeIs`` maps to the type of the second parameter after ``cls`` or + ``self``. + + In short, the form ``def foo(arg: TypeA) -> TypeIs[TypeB]: ...``, + means that if ``foo(arg)`` returns ``True``, then ``arg`` is an instance + of ``TypeB``, and if it returns ``False``, it is not an instance of ``TypeB``. + + ``TypeIs`` also works with type variables. For more information, see + :pep:`742` (Narrowing types with ``TypeIs``). + + .. versionadded:: 3.13 + + +.. data:: TypeGuard + + Special typing construct for marking user-defined type predicate functions. + + Type predicate functions are user-defined functions that return whether their + argument is an instance of a particular type. + ``TypeGuard`` works similarly to :data:`TypeIs`, but has subtly different + effects on type checking behavior (see below). Using ``-> TypeGuard`` tells the static type checker that for a given function: @@ -1419,6 +1485,8 @@ These can be used as types in annotations. They all support subscription using 2. If the return value is ``True``, the type of its argument is the type inside ``TypeGuard``. + ``TypeGuard`` also works with type variables. See :pep:`647` for more details. + For example:: def is_str_list(val: list[object]) -> TypeGuard[list[str]]: @@ -1433,23 +1501,19 @@ These can be used as types in annotations. They all support subscription using # Type of ``val`` remains as ``list[object]``. print("Not a list of strings!") - If ``is_str_list`` is a class or instance method, then the type in - ``TypeGuard`` maps to the type of the second parameter after ``cls`` or - ``self``. - - In short, the form ``def foo(arg: TypeA) -> TypeGuard[TypeB]: ...``, - means that if ``foo(arg)`` returns ``True``, then ``arg`` narrows from - ``TypeA`` to ``TypeB``. - - .. note:: - - ``TypeB`` need not be a narrower form of ``TypeA`` -- it can even be a - wider form. The main reason is to allow for things like - narrowing ``list[object]`` to ``list[str]`` even though the latter - is not a subtype of the former, since ``list`` is invariant. - The responsibility of writing type-safe type guards is left to the user. - - ``TypeGuard`` also works with type variables. See :pep:`647` for more details. + ``TypeIs`` and ``TypeGuard`` differ in the following ways: + + * ``TypeIs`` requires the narrowed type to be a subtype of the input type, while + ``TypeGuard`` does not. The main reason is to allow for things like + narrowing ``list[object]`` to ``list[str]`` even though the latter + is not a subtype of the former, since ``list`` is invariant. + * When a ``TypeGuard`` function returns ``True``, type checkers narrow the type of the + variable to exactly the ``TypeGuard`` type. When a ``TypeIs`` function returns ``True``, + type checkers can infer a more precise type combining the previously known type of the + variable with the ``TypeIs`` type. (Technically, this is known as an intersection type.) + * When a ``TypeGuard`` function returns ``False``, type checkers cannot narrow the type of + the variable at all. When a ``TypeIs`` function returns ``False``, type checkers can narrow + the type of the variable to exclude the ``TypeIs`` type. .. versionadded:: 3.10 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 72b3a4c951eda6..707dcaa160d653 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -87,6 +87,10 @@ Interpreter improvements: Performance improvements are modest -- we expect to be improving this over the next few releases. +New typing features: + +* :pep:`742`: :data:`typing.TypeIs` was added, providing more intuitive + type narrowing behavior. New Features ============ diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 927f74eb69fbc7..bae0a8480b994f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -38,7 +38,7 @@ from typing import Self, LiteralString from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs -from typing import TypeGuard +from typing import TypeGuard, TypeIs import abc import textwrap import typing @@ -5207,6 +5207,7 @@ def test_subclass_special_form(self): Literal[1, 2], Concatenate[int, ParamSpec("P")], TypeGuard[int], + TypeIs[range], ): with self.subTest(msg=obj): with self.assertRaisesRegex( @@ -6748,6 +6749,7 @@ class C(Generic[T]): pass self.assertEqual(get_args(NotRequired[int]), (int,)) self.assertEqual(get_args(TypeAlias), ()) self.assertEqual(get_args(TypeGuard[int]), (int,)) + self.assertEqual(get_args(TypeIs[range]), (range,)) Ts = TypeVarTuple('Ts') self.assertEqual(get_args(Ts), ()) self.assertEqual(get_args((*Ts,)[0]), (Ts,)) @@ -9592,6 +9594,56 @@ def test_no_isinstance(self): issubclass(int, TypeGuard) +class TypeIsTests(BaseTestCase): + def test_basics(self): + TypeIs[int] # OK + + def foo(arg) -> TypeIs[int]: ... + self.assertEqual(gth(foo), {'return': TypeIs[int]}) + + with self.assertRaises(TypeError): + TypeIs[int, str] + + def test_repr(self): + self.assertEqual(repr(TypeIs), 'typing.TypeIs') + cv = TypeIs[int] + self.assertEqual(repr(cv), 'typing.TypeIs[int]') + cv = TypeIs[Employee] + self.assertEqual(repr(cv), 'typing.TypeIs[%s.Employee]' % __name__) + cv = TypeIs[tuple[int]] + self.assertEqual(repr(cv), 'typing.TypeIs[tuple[int]]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(TypeIs)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(TypeIs[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs'): + class E(TypeIs): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs\[int\]'): + class F(TypeIs[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + TypeIs() + with self.assertRaises(TypeError): + type(TypeIs)() + with self.assertRaises(TypeError): + type(TypeIs[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, TypeIs[int]) + with self.assertRaises(TypeError): + issubclass(int, TypeIs) + + SpecialAttrsP = typing.ParamSpec('SpecialAttrsP') SpecialAttrsT = typing.TypeVar('SpecialAttrsT', int, float, complex) @@ -9691,6 +9743,7 @@ def test_special_attrs(self): typing.Optional: 'Optional', typing.TypeAlias: 'TypeAlias', typing.TypeGuard: 'TypeGuard', + typing.TypeIs: 'TypeIs', typing.TypeVar: 'TypeVar', typing.Union: 'Union', typing.Self: 'Self', @@ -9705,6 +9758,7 @@ def test_special_attrs(self): typing.Literal[True, 2]: 'Literal', typing.Optional[Any]: 'Optional', typing.TypeGuard[Any]: 'TypeGuard', + typing.TypeIs[Any]: 'TypeIs', typing.Union[Any]: 'Any', typing.Union[int, float]: 'Union', # Incompatible special forms (tested in test_special_attrs2) diff --git a/Lib/typing.py b/Lib/typing.py index d8e4ee3635994c..231492cdcc01cf 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -153,6 +153,7 @@ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TypeIs', 'TypeAliasType', 'Unpack', ] @@ -818,28 +819,31 @@ def Concatenate(self, parameters): @_SpecialForm def TypeGuard(self, parameters): - """Special typing construct for marking user-defined type guard functions. + """Special typing construct for marking user-defined type predicate functions. ``TypeGuard`` can be used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. + type predicate function. ``TypeGuard`` only accepts a single type argument. At runtime, functions marked this way should return a boolean. ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard". + conditional expression here is sometimes referred to as a "type predicate". Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeGuard[...]`` or + ``TypeIs[...]`` as its return type to alert static type checkers to + this intention. ``TypeGuard`` should be used over ``TypeIs`` when narrowing + from an incompatible type (e.g., ``list[object]`` to ``list[int]``) or when + the function does not return ``True`` for all instances of the narrowed type. - Using ``-> TypeGuard`` tells the static type checker that for a given - function: + Using ``-> TypeGuard[NarrowedType]`` tells the static type checker that + for a given function: 1. The return value is a boolean. 2. If the return value is ``True``, the type of its argument - is the type inside ``TypeGuard``. + is ``NarrowedType``. For example:: @@ -860,7 +864,7 @@ def func1(val: list[object]): type-unsafe results. The main reason is to allow for things like narrowing ``list[object]`` to ``list[str]`` even though the latter is not a subtype of the former, since ``list`` is invariant. The responsibility of - writing type-safe type guards is left to the user. + writing type-safe type predicates is left to the user. ``TypeGuard`` also works with type variables. For more information, see PEP 647 (User-Defined Type Guards). @@ -869,6 +873,75 @@ def func1(val: list[object]): return _GenericAlias(self, (item,)) +@_SpecialForm +def TypeIs(self, parameters): + """Special typing construct for marking user-defined type predicate functions. + + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and accept + at least one argument. + + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type predicate". + + Sometimes it would be convenient to use a user-defined boolean function + as a type predicate. Such a function should use ``TypeIs[...]`` or + ``TypeGuard[...]`` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. + + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for + a given function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and + ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. + + For example:: + + from typing import assert_type, final, TypeIs + + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass + + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) + + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + ``TypeIs`` also works with type variables. For more information, see + PEP 742 (Narrowing types with ``TypeIs``). + """ + item = _type_check(parameters, f'{self} accepts only single type.') + return _GenericAlias(self, (item,)) + + class ForwardRef(_Final, _root=True): """Internal wrapper to hold a forward reference.""" @@ -1241,11 +1314,12 @@ class _GenericAlias(_BaseGenericAlias, _root=True): # A = Callable[[], None] # _CallableGenericAlias # B = Callable[[T], None] # _CallableGenericAlias # C = B[int] # _CallableGenericAlias - # * Parameterized `Final`, `ClassVar` and `TypeGuard`: + # * Parameterized `Final`, `ClassVar`, `TypeGuard`, and `TypeIs`: # # All _GenericAlias # Final[int] # ClassVar[float] - # TypeVar[bool] + # TypeGuard[bool] + # TypeIs[range] def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) diff --git a/Misc/NEWS.d/next/Library/2024-04-03-16-01-31.gh-issue-117516.7DlHje.rst b/Misc/NEWS.d/next/Library/2024-04-03-16-01-31.gh-issue-117516.7DlHje.rst new file mode 100644 index 00000000000000..bbf69126d956d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-03-16-01-31.gh-issue-117516.7DlHje.rst @@ -0,0 +1 @@ +Add :data:`typing.TypeIs`, implementing :pep:`742`. Patch by Jelle Zijlstra. From 22b25d1ebaab7b8c4833a8c120c8b4699a830f40 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 9 Apr 2024 12:40:58 +0100 Subject: [PATCH 125/143] gh-116622: Enable `test_doctest` on platforms that don't support subprocesses (#116758) Co-authored-by: Nikita Sobolev --- Lib/test/test_doctest/test_doctest.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index dd8cc9be3a4a8a..0a2a016fff13e5 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -18,8 +18,12 @@ import contextlib -if not support.has_subprocess_support: - raise unittest.SkipTest("test_CLI requires subprocess support.") +def doctest_skip_if(condition): + def decorator(func): + if condition and support.HAVE_DOCSTRINGS: + func.__doc__ = ">>> pass # doctest: +SKIP" + return func + return decorator # NOTE: There are some additional tests relating to interaction with @@ -466,7 +470,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -2966,6 +2970,7 @@ def test_unicode(): """ TestResults(failed=1, attempted=1) """ +@doctest_skip_if(not support.has_subprocess_support) def test_CLI(): r""" The doctest module can be used to run doctests against an arbitrary file. These tests test this CLI functionality. From fa58e75a8605146a89ef72b58b4529669ac48366 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 9 Apr 2024 08:17:28 -0700 Subject: [PATCH 126/143] gh-116720: Fix corner cases of taskgroups (#117407) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prevents external cancellations of a task group's parent task to be dropped when an internal cancellation happens at the same time. Also strengthen the semantics of uncancel() to clear self._must_cancel when the cancellation count reaches zero. Co-Authored-By: Tin Tvrtković Co-Authored-By: Arthur Tacca --- Doc/library/asyncio-task.rst | 30 +++++++++ Doc/whatsnew/3.13.rst | 34 ++++++++-- Lib/asyncio/taskgroups.py | 19 ++++-- Lib/asyncio/tasks.py | 2 + Lib/test/test_asyncio/test_taskgroups.py | 66 +++++++++++++++++++ Lib/test/test_asyncio/test_tasks.py | 24 +++++++ ...-04-04-15-28-12.gh-issue-116720.aGhXns.rst | 18 +++++ Modules/_asynciomodule.c | 3 + 8 files changed, 183 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-04-15-28-12.gh-issue-116720.aGhXns.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 3b10a0d628a86e..3d300c37419f13 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -392,6 +392,27 @@ is also included in the exception group. The same special case is made for :exc:`KeyboardInterrupt` and :exc:`SystemExit` as in the previous paragraph. +Task groups are careful not to mix up the internal cancellation used to +"wake up" their :meth:`~object.__aexit__` with cancellation requests +for the task in which they are running made by other parties. +In particular, when one task group is syntactically nested in another, +and both experience an exception in one of their child tasks simultaneously, +the inner task group will process its exceptions, and then the outer task group +will receive another cancellation and process its own exceptions. + +In the case where a task group is cancelled externally and also must +raise an :exc:`ExceptionGroup`, it will call the parent task's +:meth:`~asyncio.Task.cancel` method. This ensures that a +:exc:`asyncio.CancelledError` will be raised at the next +:keyword:`await`, so the cancellation is not lost. + +Task groups preserve the cancellation count +reported by :meth:`asyncio.Task.cancelling`. + +.. versionchanged:: 3.13 + + Improved handling of simultaneous internal and external cancellations + and correct preservation of cancellation counts. Sleeping ======== @@ -1369,6 +1390,15 @@ Task Object catching :exc:`CancelledError`, it needs to call this method to remove the cancellation state. + When this method decrements the cancellation count to zero, + the method checks if a previous :meth:`cancel` call had arranged + for :exc:`CancelledError` to be thrown into the task. + If it hasn't been thrown yet, that arrangement will be + rescinded (by resetting the internal ``_must_cancel`` flag). + + .. versionchanged:: 3.13 + Changed to rescind pending cancellation requests upon reaching zero. + .. method:: cancelling() Return the number of pending cancellation requests to this Task, i.e., diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 707dcaa160d653..d971beb27bbf27 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -196,13 +196,6 @@ Other Language Changes (Contributed by Sebastian Pipping in :gh:`115623`.) -* When :func:`asyncio.TaskGroup.create_task` is called on an inactive - :class:`asyncio.TaskGroup`, the given coroutine will be closed (which - prevents a :exc:`RuntimeWarning` about the given coroutine being - never awaited). - - (Contributed by Arthur Tacca and Jason Zhang in :gh:`115957`.) - * The :func:`ssl.create_default_context` API now includes :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`ssl.VERIFY_X509_STRICT` in its default flags. @@ -300,6 +293,33 @@ asyncio with the tasks being completed. (Contributed by Justin Arthur in :gh:`77714`.) +* When :func:`asyncio.TaskGroup.create_task` is called on an inactive + :class:`asyncio.TaskGroup`, the given coroutine will be closed (which + prevents a :exc:`RuntimeWarning` about the given coroutine being + never awaited). + (Contributed by Arthur Tacca and Jason Zhang in :gh:`115957`.) + +* Improved behavior of :class:`asyncio.TaskGroup` when an external cancellation + collides with an internal cancellation. For example, when two task groups + are nested and both experience an exception in a child task simultaneously, + it was possible that the outer task group would hang, because its internal + cancellation was swallowed by the inner task group. + + In the case where a task group is cancelled externally and also must + raise an :exc:`ExceptionGroup`, it will now call the parent task's + :meth:`~asyncio.Task.cancel` method. This ensures that a + :exc:`asyncio.CancelledError` will be raised at the next + :keyword:`await`, so the cancellation is not lost. + + An added benefit of these changes is that task groups now preserve the + cancellation count (:meth:`asyncio.Task.cancelling`). + + In order to handle some corner cases, :meth:`asyncio.Task.uncancel` may now + reset the undocumented ``_must_cancel`` flag when the cancellation count + reaches zero. + + (Inspired by an issue reported by Arthur Tacca in :gh:`116720`.) + * Add :meth:`asyncio.Queue.shutdown` (along with :exc:`asyncio.QueueShutDown`) for queue termination. (Contributed by Laurie Opperman and Yves Duprat in :gh:`104228`.) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 57f01230159319..f2ee9648c43876 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -77,12 +77,6 @@ async def __aexit__(self, et, exc, tb): propagate_cancellation_error = exc else: propagate_cancellation_error = None - if self._parent_cancel_requested: - # If this flag is set we *must* call uncancel(). - if self._parent_task.uncancel() == 0: - # If there are no pending cancellations left, - # don't propagate CancelledError. - propagate_cancellation_error = None if et is not None: if not self._aborting: @@ -130,6 +124,13 @@ async def __aexit__(self, et, exc, tb): if self._base_error is not None: raise self._base_error + if self._parent_cancel_requested: + # If this flag is set we *must* call uncancel(). + if self._parent_task.uncancel() == 0: + # If there are no pending cancellations left, + # don't propagate CancelledError. + propagate_cancellation_error = None + # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. if propagate_cancellation_error is not None and not self._errors: @@ -139,6 +140,12 @@ async def __aexit__(self, et, exc, tb): self._errors.append(exc) if self._errors: + # If the parent task is being cancelled from the outside + # of the taskgroup, un-cancel and re-cancel the parent task, + # which will keep the cancel count stable. + if self._parent_task.cancelling(): + self._parent_task.uncancel() + self._parent_task.cancel() # Exceptions are heavy objects that can have object # cycles (bad for GC); let's not keep a reference to # a bunch of them. diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 7fb697b9441c33..dadcb5b5f36bd7 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -255,6 +255,8 @@ def uncancel(self): """ if self._num_cancels_requested > 0: self._num_cancels_requested -= 1 + if self._num_cancels_requested == 0: + self._must_cancel = False return self._num_cancels_requested def __eager_start(self): diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 1ec8116953f811..4852536defc93d 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -833,6 +833,72 @@ async def run_coro_after_tg_closes(): loop = asyncio.get_event_loop() loop.run_until_complete(run_coro_after_tg_closes()) + async def test_cancelling_level_preserved(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(raise_after(0.0, RuntimeError)) + except* RuntimeError: + pass + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_nested_groups_both_cancelled(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as outer_tg: + try: + async with asyncio.TaskGroup() as inner_tg: + inner_tg.create_task(raise_after(0, RuntimeError)) + outer_tg.create_task(raise_after(0, ValueError)) + except* RuntimeError: + pass + else: + self.fail("RuntimeError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 1) + except* ValueError: + pass + else: + self.fail("ValueError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_error_and_cancel(self): + event = asyncio.Event() + + async def raise_error(): + event.set() + await asyncio.sleep(0) + raise RuntimeError() + + async def inner(): + try: + async with taskgroups.TaskGroup() as tg: + tg.create_task(raise_error()) + await asyncio.sleep(1) + self.fail("Sleep in group should have been cancelled") + except* RuntimeError: + self.assertEqual(asyncio.current_task().cancelling(), 1) + self.assertEqual(asyncio.current_task().cancelling(), 1) + await asyncio.sleep(1) + self.fail("Sleep after group should have been cancelled") + + async def outer(): + t = asyncio.create_task(inner()) + await event.wait() + self.assertEqual(t.cancelling(), 0) + t.cancel() + self.assertEqual(t.cancelling(), 1) + with self.assertRaises(asyncio.CancelledError): + await t + self.assertTrue(t.cancelled()) + + await outer() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index bc6d88e65a4966..5b09c81faef62a 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -684,6 +684,30 @@ def on_timeout(): finally: loop.close() + def test_uncancel_resets_must_cancel(self): + + async def coro(): + await fut + return 42 + + loop = asyncio.new_event_loop() + fut = asyncio.Future(loop=loop) + task = self.new_task(loop, coro()) + loop.run_until_complete(asyncio.sleep(0)) # Get task waiting for fut + fut.set_result(None) # Make task runnable + try: + task.cancel() # Enter cancelled state + self.assertEqual(task.cancelling(), 1) + self.assertTrue(task._must_cancel) + + task.uncancel() # Undo cancellation + self.assertEqual(task.cancelling(), 0) + self.assertFalse(task._must_cancel) + finally: + res = loop.run_until_complete(task) + self.assertEqual(res, 42) + loop.close() + def test_cancel(self): def gen(): diff --git a/Misc/NEWS.d/next/Library/2024-04-04-15-28-12.gh-issue-116720.aGhXns.rst b/Misc/NEWS.d/next/Library/2024-04-04-15-28-12.gh-issue-116720.aGhXns.rst new file mode 100644 index 00000000000000..39c7d6b8a1e978 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-04-15-28-12.gh-issue-116720.aGhXns.rst @@ -0,0 +1,18 @@ +Improved behavior of :class:`asyncio.TaskGroup` when an external cancellation +collides with an internal cancellation. For example, when two task groups +are nested and both experience an exception in a child task simultaneously, +it was possible that the outer task group would misbehave, because +its internal cancellation was swallowed by the inner task group. + +In the case where a task group is cancelled externally and also must +raise an :exc:`ExceptionGroup`, it will now call the parent task's +:meth:`~asyncio.Task.cancel` method. This ensures that a +:exc:`asyncio.CancelledError` will be raised at the next +:keyword:`await`, so the cancellation is not lost. + +An added benefit of these changes is that task groups now preserve the +cancellation count (:meth:`asyncio.Task.cancelling`). + +In order to handle some corner cases, :meth:`asyncio.Task.uncancel` may now +reset the undocumented ``_must_cancel`` flag when the cancellation count +reaches zero. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 29246cfa6afd00..b886051186de9c 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2393,6 +2393,9 @@ _asyncio_Task_uncancel_impl(TaskObj *self) { if (self->task_num_cancels_requested > 0) { self->task_num_cancels_requested -= 1; + if (self->task_num_cancels_requested == 0) { + self->task_must_cancel = 0; + } } return PyLong_FromLong(self->task_num_cancels_requested); } From 6edde8a91c753dba03c92315b7585209931c704b Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 9 Apr 2024 11:50:49 -0400 Subject: [PATCH 127/143] gh-117658: Fix check_dump_traceback_threads in free-threaded build (#117659) With the GIL disabled, the waiting thread may still be in the `self.running.set() ` call when faulthandler dumps tracebacks. --- Lib/test/test_faulthandler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 200f34d18ca60a..61ec8fe3151af1 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -575,10 +575,12 @@ def run(self): lineno = 8 else: lineno = 10 + # When the traceback is dumped, the waiter thread may be in the + # `self.running.set()` call or in `self.stop.wait()`. regex = r""" ^Thread 0x[0-9a-f]+ \(most recent call first\): (?: File ".*threading.py", line [0-9]+ in [_a-z]+ - ){{1,3}} File "", line 23 in run + ){{1,3}} File "", line (?:22|23) in run File ".*threading.py", line [0-9]+ in _bootstrap_inner File ".*threading.py", line [0-9]+ in _bootstrap From a25c02eaf01abc7ca79efdbcda986b9cc2787b6c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 9 Apr 2024 18:26:35 +0200 Subject: [PATCH 128/143] Revert "gh-106023: Update What's New in 3.13: _PyObject_FastCall() (#117633)" (#117676) This reverts commit 9a12f5d1c19dee1f89684be776680aeaf117be5b. I was wrong: the _PyObject_FastCall() function was removed. But we kept the _PyObject_FastCallDict() function. --- Doc/whatsnew/3.13.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d971beb27bbf27..d394fbe3b0c357 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -2030,6 +2030,11 @@ Removed (Contributed by Victor Stinner in :gh:`105182`.) +* Remove private ``_PyObject_FastCall()`` function: + use ``PyObject_Vectorcall()`` which is available since Python 3.8 + (:pep:`590`). + (Contributed by Victor Stinner in :gh:`106023`.) + * Remove ``cpython/pytime.h`` header file: it only contained private functions. (Contributed by Victor Stinner in :gh:`106316`.) From d5f1139c79525b4e7e4e8ad8c3e5fb831bbc3f28 Mon Sep 17 00:00:00 2001 From: Vlad4896 <166005126+Vlad4896@users.noreply.github.com> Date: Tue, 9 Apr 2024 20:53:00 +0300 Subject: [PATCH 129/143] gh-117534: Add checking for input parameter in iso_to_ymd (#117543) Moves the validation for invalid years in the C implementation of the `datetime` module into a common location between `fromisoformat` and `fromisocalendar`, which improves the error message and fixes a failed assertion when parsing invalid ISO 8601 years using one of the "ISO weeks" formats. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/test/datetimetester.py | 4 ++++ Misc/ACKS | 1 + ...024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst | 2 ++ Modules/_datetimemodule.c | 16 +++++++++------- 4 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index c77263998c99f5..570110893629cf 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1927,6 +1927,10 @@ def test_fromisoformat_fails(self): '2009-02-29', # Invalid leap day '2019-W53-1', # No week 53 in 2019 '2020-W54-1', # No week 54 + '0000-W25-1', # Invalid year + '10000-W25-1', # Invalid year + '2020-W25-0', # Invalid day-of-week + '2020-W25-8', # Invalid day-of-week '2009\ud80002\ud80028', # Separators are surrogate codepoints ] diff --git a/Misc/ACKS b/Misc/ACKS index fe014a364dd82d..a108ec37d4425e 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -496,6 +496,7 @@ David Edelsohn John Edmonds Benjamin Edwards Grant Edwards +Vlad Efanov Zvi Effron John Ehresman Tal Einat diff --git a/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst b/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst new file mode 100644 index 00000000000000..4b7dda610fc2b2 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst @@ -0,0 +1,2 @@ +Improve validation logic in the C implementation of :meth:`datetime.fromisoformat` +to better handle invalid years. Patch by Vlad Efanov. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index a626bda2ea9be9..2c9ef4b52851b7 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -416,6 +416,10 @@ iso_week1_monday(int year) static int iso_to_ymd(const int iso_year, const int iso_week, const int iso_day, int *year, int *month, int *day) { + // Year is bounded to 0 < year < 10000 because 9999-12-31 is (9999, 52, 5) + if (iso_year < MINYEAR || iso_year > MAXYEAR) { + return -4; + } if (iso_week <= 0 || iso_week >= 53) { int out_of_range = 1; if (iso_week == 53) { @@ -762,7 +766,7 @@ parse_isoformat_date(const char *dtstr, const size_t len, int *year, int *month, * -2: Inconsistent date separator usage * -3: Failed to parse ISO week. * -4: Failed to parse ISO day. - * -5, -6: Failure in iso_to_ymd + * -5, -6, -7: Failure in iso_to_ymd */ const char *p = dtstr; p = parse_digits(p, year, 4); @@ -3142,15 +3146,13 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return NULL; } - // Year is bounded to 0 < year < 10000 because 9999-12-31 is (9999, 52, 5) - if (year < MINYEAR || year > MAXYEAR) { - PyErr_Format(PyExc_ValueError, "Year is out of range: %d", year); - return NULL; - } - int month; int rv = iso_to_ymd(year, week, day, &year, &month, &day); + if (rv == -4) { + PyErr_Format(PyExc_ValueError, "Year is out of range: %d", year); + return NULL; + } if (rv == -2) { PyErr_Format(PyExc_ValueError, "Invalid week: %d", week); From e5521bcca916c63866f0aa1c4dfb3a315d6ada73 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Tue, 9 Apr 2024 11:31:07 -0700 Subject: [PATCH 130/143] gh-117663: [Enum] fix _simple_enum's detection of aliases (GH-117664) --- Lib/enum.py | 72 +++++++++++-------- Lib/test/test_enum.py | 52 +++++++++++++- ...-04-08-19-12-26.gh-issue-117663.CPfc_p.rst | 2 + 3 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-08-19-12-26.gh-issue-117663.CPfc_p.rst diff --git a/Lib/enum.py b/Lib/enum.py index 2a135e1b1f1826..98a49eafbb9897 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1088,8 +1088,6 @@ def _add_member_(cls, name, member): setattr(cls, name, member) # now add to _member_map_ (even aliases) cls._member_map_[name] = member - # - cls._member_map_[name] = member EnumMeta = EnumType # keep EnumMeta name for backwards compatibility @@ -1802,20 +1800,31 @@ def convert_class(cls): for name, value in attrs.items(): if isinstance(value, auto) and auto.value is _auto_null: value = gnv(name, 1, len(member_names), gnv_last_values) - if value in value2member_map or value in unhashable_values: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - enum_class(value)._add_alias_(name) + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) @@ -1847,24 +1856,31 @@ def convert_class(cls): if value.value is _auto_null: value.value = gnv(name, 1, len(member_names), gnv_last_values) value = value.value + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias try: - contained = value in value2member_map + contained = value2member_map.get(member._value_) except TypeError: - contained = value in unhashable_values - if contained: + contained = None + if member._value_ in unhashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - enum_class(value)._add_alias_(name) + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 6418d243db65ce..529dfc62eff680 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -5170,7 +5170,57 @@ class Unhashable: self.assertIn('python', Unhashable) self.assertEqual(Unhashable.name.value, 'python') self.assertEqual(Unhashable.name.name, 'name') - _test_simple_enum(Unhashable, Unhashable) + _test_simple_enum(CheckedUnhashable, Unhashable) + ## + class CheckedComplexStatus(IntEnum): + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + # + @_simple_enum(IntEnum) + class ComplexStatus: + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + _test_simple_enum(CheckedComplexStatus, ComplexStatus) + # + # + class CheckedComplexFlag(IntFlag): + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'outer upper half' + PANTS = 2, 'lower half' + self.assertIs(CheckedComplexFlag.SHIRT, CheckedComplexFlag.VEST) + # + @_simple_enum(IntFlag) + class ComplexFlag: + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'uppert half' + PANTS = 2, 'lower half' + _test_simple_enum(CheckedComplexFlag, ComplexFlag) class MiscTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-04-08-19-12-26.gh-issue-117663.CPfc_p.rst b/Misc/NEWS.d/next/Library/2024-04-08-19-12-26.gh-issue-117663.CPfc_p.rst new file mode 100644 index 00000000000000..2c7a5224b5a6eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-08-19-12-26.gh-issue-117663.CPfc_p.rst @@ -0,0 +1,2 @@ +Fix ``_simple_enum`` to detect aliases when multiple arguments are present +but only one is the member value. From 54084e2fe405d2c0894fff6e5c3432c85f46f75e Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 9 Apr 2024 20:40:50 +0200 Subject: [PATCH 131/143] Post 3.13.0a6 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index c14b1811ad758d..35c595deaa72c2 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 6 /* Version as a string */ -#define PY_VERSION "3.13.0a6" +#define PY_VERSION "3.13.0a6+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From ca7591577926d13083291c3caef408116429f539 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 9 Apr 2024 22:03:47 +0200 Subject: [PATCH 132/143] gh-117648: Amend NEWS entry (#117697) Make the wording more vague; the performance impact varies a lot depending on platform and input. --- Misc/NEWS.d/3.13.0a6.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst index f7e90722d659a8..52735dba3578b5 100644 --- a/Misc/NEWS.d/3.13.0a6.rst +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -4,7 +4,7 @@ .. release date: 2024-04-09 .. section: Core and Builtins -Speedup :func:`os.path.join` by up to 6% on Windows. +Improve performance of :func:`os.path.join`. .. From a05068db0cb43337d20a936d919b9d88c35d9818 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 9 Apr 2024 22:59:45 +0200 Subject: [PATCH 133/143] gh-117597: Clarify exception handling in the tutorial (#117681) --- Doc/tutorial/errors.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 0b9acd00fdc6bd..981b14f5a4212b 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -119,9 +119,9 @@ may name multiple exceptions as a parenthesized tuple, for example:: ... except (RuntimeError, TypeError, NameError): ... pass -A class in an :keyword:`except` clause is compatible with an exception if it is -the same class or a base class thereof (but not the other way around --- an -*except clause* listing a derived class is not compatible with a base class). +A class in an :keyword:`except` clause matches exceptions which are instances of the +class itself or one of its derived classes (but not the other way around --- an +*except clause* listing a derived class does not match instances of its base classes). For example, the following code will print B, C, D in that order:: class B(Exception): From 73906d5c908c1e0b73c5436faeff7d93698fc074 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 9 Apr 2024 23:00:41 +0200 Subject: [PATCH 134/143] gh-117360: Clearer wording in os.path.lexists() docs (#117679) Co-authored-by: Zachary Ware --- Doc/library/os.path.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index dcc877da0b3122..fcf4b6d68e018c 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -145,7 +145,7 @@ the :mod:`glob` module.) .. function:: lexists(path) - Return ``True`` if *path* refers to an existing path. Returns ``True`` for + Return ``True`` if *path* refers to an existing path, including broken symbolic links. Equivalent to :func:`exists` on platforms lacking :func:`os.lstat`. From 0d42ac9474f857633d00b414c0715f4efa73f1ca Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 10 Apr 2024 10:12:05 +0200 Subject: [PATCH 135/143] gh-117431: Argument Clinic: copy forced text signature when cloning (#117591) --- Lib/test/test_clinic.py | 57 +++++++++++++++++++++++++++- Objects/clinic/unicodeobject.c.h | 10 ++--- Tools/clinic/libclinic/dsl_parser.py | 12 ++++-- Tools/clinic/libclinic/function.py | 1 + 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index a5887cdb56e3ca..e3ba3d943216de 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -5,7 +5,7 @@ from functools import partial from test import support, test_tools from test.support import os_helper -from test.support.os_helper import TESTFN, unlink +from test.support.os_helper import TESTFN, unlink, rmtree from textwrap import dedent from unittest import TestCase import inspect @@ -662,6 +662,61 @@ class C "void *" "" err = "Illegal C basename: '.illegal.'" self.expect_failure(block, err, lineno=7) + def test_cloned_forced_text_signature(self): + block = dedent(""" + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + param a + b: object = NULL + / + + docstring + [clinic start generated code]*/ + + /*[clinic input] + dst = src + [clinic start generated code]*/ + """) + self.clinic.parse(block) + self.addCleanup(rmtree, "clinic") + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + + src_docstring_lines = funcs[0].docstring.split("\n") + dst_docstring_lines = funcs[1].docstring.split("\n") + + # Signatures are copied. + self.assertEqual(src_docstring_lines[0], "src($module, a[, b])") + self.assertEqual(dst_docstring_lines[0], "dst($module, a[, b])") + + # Param docstrings are copied. + self.assertIn(" param a", src_docstring_lines) + self.assertIn(" param a", dst_docstring_lines) + + # Docstrings are not copied. + self.assertIn("docstring", src_docstring_lines) + self.assertNotIn("docstring", dst_docstring_lines) + + def test_cloned_forced_text_signature_illegal(self): + block = """ + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + b: object = NULL + / + [clinic start generated code]*/ + + /*[clinic input] + @text_signature "($module, a_override[, b])" + dst = src + [clinic start generated code]*/ + """ + err = "Cannot use @text_signature when cloning a function" + self.expect_failure(block, err, lineno=11) + class ParseFileUnitTest(TestCase): def expect_parsing_failure( diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 01c40b90d9b4b8..78e14b0021d006 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -357,7 +357,7 @@ unicode_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb } PyDoc_STRVAR(unicode_find__doc__, -"find($self, sub, start=None, end=None, /)\n" +"find($self, sub[, start[, end]], /)\n" "--\n" "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" @@ -413,7 +413,7 @@ unicode_find(PyObject *str, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(unicode_index__doc__, -"index($self, sub, start=None, end=None, /)\n" +"index($self, sub[, start[, end]], /)\n" "--\n" "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" @@ -1060,7 +1060,7 @@ unicode_removesuffix(PyObject *self, PyObject *arg) } PyDoc_STRVAR(unicode_rfind__doc__, -"rfind($self, sub, start=None, end=None, /)\n" +"rfind($self, sub[, start[, end]], /)\n" "--\n" "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" @@ -1116,7 +1116,7 @@ unicode_rfind(PyObject *str, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(unicode_rindex__doc__, -"rindex($self, sub, start=None, end=None, /)\n" +"rindex($self, sub[, start[, end]], /)\n" "--\n" "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" @@ -1888,4 +1888,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=3aa49013ffa3fa93 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9fee62bd337f809b input=a9049054013a1b77]*/ diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index 4c739efe1066e4..9e22d847c4dc90 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -666,6 +666,8 @@ def state_modulename_name(self, line: str) -> None: if equals: existing = existing.strip() if libclinic.is_legal_py_identifier(existing): + if self.forced_text_signature: + fail("Cannot use @text_signature when cloning a function") # we're cloning! names = self.parse_function_names(before) return self.parse_cloned_function(names, existing) @@ -689,7 +691,8 @@ def state_modulename_name(self, line: str) -> None: kind=self.kind, coexist=self.coexist, critical_section=self.critical_section, - target_critical_section=self.target_critical_section + target_critical_section=self.target_critical_section, + forced_text_signature=self.forced_text_signature ) self.add_function(func) @@ -1324,13 +1327,14 @@ def state_function_docstring(self, line: str) -> None: self.docstring_append(self.function, line) + @staticmethod def format_docstring_signature( - self, f: Function, parameters: list[Parameter] + f: Function, parameters: list[Parameter] ) -> str: lines = [] lines.append(f.displayname) - if self.forced_text_signature: - lines.append(self.forced_text_signature) + if f.forced_text_signature: + lines.append(f.forced_text_signature) elif f.kind in {GETTER, SETTER}: # @getter and @setter do not need signatures like a method or a function. return '' diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index 572916bbe123b4..93901263e44c04 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -107,6 +107,7 @@ class Function: # functions with optional groups because we can't represent # those accurately with inspect.Signature in 3.4. docstring_only: bool = False + forced_text_signature: str | None = None critical_section: bool = False target_critical_section: list[str] = dc.field(default_factory=list) From f90ff0367271ea474b4ce3c8e2643cb51d188c18 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 10 Apr 2024 10:28:48 +0200 Subject: [PATCH 136/143] gh-117686: Improve the performance of ntpath.expanduser() (#117690) Refactor out _get_bothseps() call from the loop. --- Lib/ntpath.py | 4 +++- Misc/NEWS.d/3.13.0a6.rst | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index da5231ff2c0931..f5d1a2195dd633 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -368,13 +368,15 @@ def expanduser(path): If user or $HOME is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): + seps = b'\\/' tilde = b'~' else: + seps = '\\/' tilde = '~' if not path.startswith(tilde): return path i, n = 1, len(path) - while i < n and path[i] not in _get_bothseps(path): + while i < n and path[i] not in seps: i += 1 if 'USERPROFILE' in os.environ: diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst index 52735dba3578b5..06807b396ed5da 100644 --- a/Misc/NEWS.d/3.13.0a6.rst +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -4,7 +4,7 @@ .. release date: 2024-04-09 .. section: Core and Builtins -Improve performance of :func:`os.path.join`. +Improve performance of :func:`os.path.join` and :func:`os.path.expanduser`. .. From 4bb7d121bc0a3fd00a3c72cd915b5dd8fac5616e Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 10 Apr 2024 12:52:47 +0300 Subject: [PATCH 137/143] gh-117692: Fix `AttributeError` in `DocTestFinder` on wrapped `builtin_or_method` (#117699) Co-authored-by: Alex Waygood --- Lib/doctest.py | 9 ++++++++- Lib/test/test_doctest/test_doctest.py | 14 ++++++++++++++ .../2024-04-09-23-22-21.gh-issue-117692.EciInD.rst | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst diff --git a/Lib/doctest.py b/Lib/doctest.py index fc0da590018b40..4e362cbb9c9d6b 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1140,7 +1140,14 @@ def _find_lineno(self, obj, source_lines): obj = obj.fget if inspect.isfunction(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. - obj = inspect.unwrap(obj).__code__ + obj = inspect.unwrap(obj) + try: + obj = obj.__code__ + except AttributeError: + # Functions implemented in C don't necessarily + # have a __code__ attribute. + # If there's no code, there's no lineno + return None if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 0a2a016fff13e5..f71d62cc174d6b 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2553,6 +2553,20 @@ def test_look_in_unwrapped(): 'one other test' """ +@doctest_skip_if(support.check_impl_detail(cpython=False)) +def test_wrapped_c_func(): + """ + # https://github.com/python/cpython/issues/117692 + >>> import binascii + >>> from test.test_doctest.decorator_mod import decorator + + >>> c_func_wrapped = decorator(binascii.b2a_hex) + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(c_func_wrapped) + >>> for test in tests: + ... print(test.lineno, test.name) + None b2a_hex + """ + def test_unittest_reportflags(): """Default unittest reporting flags can be set to control reporting diff --git a/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst b/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst new file mode 100644 index 00000000000000..98a6e125c440ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst @@ -0,0 +1,2 @@ +Fixes a bug when :class:`doctest.DocTestFinder` was failing on wrapped +``builtin_function_or_method``. From ef4118222b6d5f4532a2f1e234ba03955348d2a1 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 10 Apr 2024 20:00:01 +0900 Subject: [PATCH 138/143] gh-117142: Port _ctypes to multi-phase init (GH-117181) --- Lib/test/test_ctypes/test_refcounts.py | 14 + ...-03-29-12-21-40.gh-issue-117142.U0agfh.rst | 1 + Modules/_ctypes/_ctypes.c | 582 ++++++++++++----- Modules/_ctypes/callbacks.c | 10 +- Modules/_ctypes/callproc.c | 29 +- Modules/_ctypes/cfield.c | 4 +- Modules/_ctypes/clinic/_ctypes.c.h | 610 ++++++++++++++++++ Modules/_ctypes/ctypes.h | 46 +- Modules/_ctypes/stgdict.c | 6 +- Tools/c-analyzer/cpython/ignored.tsv | 1 + 10 files changed, 1125 insertions(+), 178 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst create mode 100644 Modules/_ctypes/clinic/_ctypes.c.h diff --git a/Lib/test/test_ctypes/test_refcounts.py b/Lib/test/test_ctypes/test_refcounts.py index e6427d4a295b15..012722d8486218 100644 --- a/Lib/test/test_ctypes/test_refcounts.py +++ b/Lib/test/test_ctypes/test_refcounts.py @@ -4,6 +4,7 @@ import unittest from test import support from test.support import import_helper +from test.support import script_helper _ctypes_test = import_helper.import_module("_ctypes_test") @@ -110,5 +111,18 @@ def func(): func() +class ModuleIsolationTest(unittest.TestCase): + def test_finalize(self): + # check if gc_decref() succeeds + script = ( + "import ctypes;" + "import sys;" + "del sys.modules['_ctypes'];" + "import _ctypes;" + "exit()" + ) + script_helper.assert_python_ok("-c", script) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst b/Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst new file mode 100644 index 00000000000000..36810bd815c502 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst @@ -0,0 +1 @@ +Convert :mod:`!_ctypes` to multi-phase initialisation (:pep:`489`). diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 631f82879311bf..3cb0b24668eb2a 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -126,8 +126,16 @@ bytes(cdata) #include "pycore_long.h" // _PyLong_GetZero() -ctypes_state global_state = {0}; +/*[clinic input] +module _ctypes +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ +#define clinic_state() (get_module_state_by_class(cls)) +#define clinic_state_sub() (get_module_state_by_class(cls->tp_base)) +#include "clinic/_ctypes.c.h" +#undef clinic_state +#undef clinic_state_sub /****************************************************************/ @@ -438,10 +446,15 @@ static PyType_Spec structparam_spec = { CType_Type - a base metaclass. Its instances (classes) have a StgInfo. */ +/*[clinic input] +class _ctypes.CType_Type "PyObject *" "clinic_state()->CType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8389fc5b74a84f2a]*/ + static int CType_Type_traverse(PyObject *self, visitproc visit, void *arg) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); if (st && st->PyCType_Type) { StgInfo *info; if (PyStgInfo_FromType(st, self, &info) < 0) { @@ -475,7 +488,7 @@ ctype_clear_stginfo(StgInfo *info) static int CType_Type_clear(PyObject *self) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); if (st && st->PyCType_Type) { StgInfo *info; if (PyStgInfo_FromType(st, self, &info) < 0) { @@ -491,8 +504,7 @@ CType_Type_clear(PyObject *self) static void CType_Type_dealloc(PyObject *self) { - ctypes_state *st = GLOBAL_STATE(); - + ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); if (st && st->PyCType_Type) { StgInfo *info; if (PyStgInfo_FromType(st, self, &info) < 0) { @@ -508,19 +520,27 @@ CType_Type_dealloc(PyObject *self) ctype_clear_stginfo(info); } } - PyTypeObject *tp = Py_TYPE(self); PyType_Type.tp_dealloc(self); Py_DECREF(tp); } +/*[clinic input] +_ctypes.CType_Type.__sizeof__ + + cls: defining_class + / +Return memory consumption of the type object. +[clinic start generated code]*/ + static PyObject * -CType_Type_sizeof(PyObject *self) +_ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=c68c235be84d03f3 input=d064433b6110d1ce]*/ { Py_ssize_t size = Py_TYPE(self)->tp_basicsize; size += Py_TYPE(self)->tp_itemsize * Py_SIZE(self); - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); StgInfo *info; if (PyStgInfo_FromType(st, self, &info) < 0) { return NULL; @@ -543,8 +563,7 @@ CType_Type_repeat(PyObject *self, Py_ssize_t length); static PyMethodDef ctype_methods[] = { - {"__sizeof__", _PyCFunction_CAST(CType_Type_sizeof), - METH_NOARGS, PyDoc_STR("Return memory consumption of the type object.")}, + _CTYPES_CTYPE_TYPE___SIZEOF___METHODDEF {0}, }; @@ -647,7 +666,7 @@ StructUnionType_init(PyObject *self, PyObject *args, PyObject *kwds, int isStruc return -1; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); StgInfo *info = PyStgInfo_Init(st, (PyTypeObject *)self); if (!info) { Py_DECREF(attrdict); @@ -710,11 +729,29 @@ UnionType_init(PyObject *self, PyObject *args, PyObject *kwds) return StructUnionType_init(self, args, kwds, 0); } -PyDoc_STRVAR(from_address_doc, -"C.from_address(integer) -> C instance\naccess a C instance at the specified address"); +/*[clinic input] +class _ctypes.CDataType "PyObject *" "clinic_state()->CType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=466a505a93d73156]*/ + + +/*[clinic input] +_ctypes.CDataType.from_address as CDataType_from_address + + type: self + cls: defining_class + value: object + / + +C.from_address(integer) -> C instance + +Access a C instance at the specified address. +[clinic start generated code]*/ static PyObject * -CDataType_from_address(PyObject *type, PyObject *value) +CDataType_from_address_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=5be4a7c0d9aa6c74 input=827a22cefe380c01]*/ { void *buf; if (!PyLong_Check(value)) { @@ -725,26 +762,37 @@ CDataType_from_address(PyObject *type, PyObject *value) buf = (void *)PyLong_AsVoidPtr(value); if (PyErr_Occurred()) return NULL; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); return PyCData_AtAddress(st, type, buf); } -PyDoc_STRVAR(from_buffer_doc, -"C.from_buffer(object, offset=0) -> C instance\ncreate a C instance from a writeable buffer"); - static int KeepRef(CDataObject *target, Py_ssize_t index, PyObject *keep); +/*[clinic input] +_ctypes.CDataType.from_buffer as CDataType_from_buffer + + type: self + cls: defining_class + obj: object + offset: Py_ssize_t = 0 + / + +C.from_buffer(object, offset=0) -> C instance + +Create a C instance from a writeable buffer. +[clinic start generated code]*/ + static PyObject * -CDataType_from_buffer(PyObject *type, PyObject *args) +CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, + Py_ssize_t offset) +/*[clinic end generated code: output=57604e99635abd31 input=0f36cedd105ca28d]*/ { - PyObject *obj; PyObject *mv; PyObject *result; Py_buffer *buffer; - Py_ssize_t offset = 0; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); StgInfo *info; if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; @@ -754,9 +802,6 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return NULL; } - if (!PyArg_ParseTuple(args, "O|n:from_buffer", &obj, &offset)) - return NULL; - mv = PyMemoryView_FromObject(obj); if (mv == NULL) return NULL; @@ -813,9 +858,6 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return result; } -PyDoc_STRVAR(from_buffer_copy_doc, -"C.from_buffer_copy(object, offset=0) -> C instance\ncreate a C instance from a readable buffer"); - static inline PyObject * generic_pycdata_new(ctypes_state *st, PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -823,14 +865,28 @@ generic_pycdata_new(ctypes_state *st, static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +/*[clinic input] +_ctypes.CDataType.from_buffer_copy as CDataType_from_buffer_copy + + type: self + cls: defining_class + buffer: Py_buffer + offset: Py_ssize_t = 0 + / + +C.from_buffer_copy(object, offset=0) -> C instance + +Create a C instance from a readable buffer. +[clinic start generated code]*/ + static PyObject * -CDataType_from_buffer_copy(PyObject *type, PyObject *args) +CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, + Py_buffer *buffer, Py_ssize_t offset) +/*[clinic end generated code: output=c8fc62b03e5cc6fa input=2a81e11b765a6253]*/ { - Py_buffer buffer; - Py_ssize_t offset = 0; PyObject *result; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); StgInfo *info; if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; @@ -840,54 +896,56 @@ CDataType_from_buffer_copy(PyObject *type, PyObject *args) return NULL; } - if (!PyArg_ParseTuple(args, "y*|n:from_buffer_copy", &buffer, &offset)) - return NULL; - if (offset < 0) { PyErr_SetString(PyExc_ValueError, "offset cannot be negative"); - PyBuffer_Release(&buffer); return NULL; } - if (info->size > buffer.len - offset) { + if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small (%zd instead of at least %zd bytes)", - buffer.len, info->size + offset); - PyBuffer_Release(&buffer); + buffer->len, info->size + offset); return NULL; } if (PySys_Audit("ctypes.cdata/buffer", "nnn", - (Py_ssize_t)buffer.buf, buffer.len, offset) < 0) { - PyBuffer_Release(&buffer); + (Py_ssize_t)buffer->buf, buffer->len, offset) < 0) { return NULL; } result = generic_pycdata_new(st, (PyTypeObject *)type, NULL, NULL); if (result != NULL) { memcpy(((CDataObject *)result)->b_ptr, - (char *)buffer.buf + offset, info->size); + (char *)buffer->buf + offset, info->size); } - PyBuffer_Release(&buffer); return result; } -PyDoc_STRVAR(in_dll_doc, -"C.in_dll(dll, name) -> C instance\naccess a C instance in a dll"); +/*[clinic input] +_ctypes.CDataType.in_dll as CDataType_in_dll + + type: self + cls: defining_class + dll: object + name: str + / + +C.in_dll(dll, name) -> C instance + +Access a C instance in a dll. +[clinic start generated code]*/ static PyObject * -CDataType_in_dll(PyObject *type, PyObject *args) +CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, + const char *name) +/*[clinic end generated code: output=d0e5c43b66bfa21f input=f85bf281477042b4]*/ { - PyObject *dll; - char *name; PyObject *obj; void *handle; void *address; - if (!PyArg_ParseTuple(args, "Os:in_dll", &dll, &name)) - return NULL; - if (PySys_Audit("ctypes.dlsym", "O", args) < 0) { + if (PySys_Audit("ctypes.dlsym", "Os", dll, name) < 0) { return NULL; } @@ -932,15 +990,24 @@ CDataType_in_dll(PyObject *type, PyObject *args) return NULL; } #endif - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); return PyCData_AtAddress(st, type, address); } -PyDoc_STRVAR(from_param_doc, -"Convert a Python object into a function call parameter."); +/*[clinic input] +_ctypes.CDataType.from_param as CDataType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ static PyObject * -CDataType_from_param(PyObject *type, PyObject *value) +CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=8da9e34263309f9e input=275a52c4899ddff0]*/ { PyObject *as_parameter; int res = PyObject_IsInstance(value, type); @@ -949,7 +1016,7 @@ CDataType_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); if (PyCArg_CheckExact(st, value)) { PyCArgObject *p = (PyCArgObject *)value; PyObject *ob = p->obj; @@ -979,7 +1046,7 @@ CDataType_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = CDataType_from_param(type, as_parameter); + value = CDataType_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -991,11 +1058,11 @@ CDataType_from_param(PyObject *type, PyObject *value) } static PyMethodDef CDataType_methods[] = { - { "from_param", CDataType_from_param, METH_O, from_param_doc }, - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc }, + CDATATYPE_FROM_PARAM_METHODDEF + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF { NULL, NULL }, }; @@ -1006,10 +1073,11 @@ CType_Type_repeat(PyObject *self, Py_ssize_t length) return PyErr_Format(PyExc_ValueError, "Array length must be >= 0, not %zd", length); - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); return PyCArrayType_from_ctype(st, self, length); } + static int PyCStructType_setattro(PyObject *self, PyObject *key, PyObject *value) { @@ -1083,6 +1151,12 @@ size property/method, and the sequence protocol. */ +/*[clinic input] +class _ctypes.PyCPointerType "PyObject *" "clinic_state()->PyCPointerType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c45e96c1f7645ab7]*/ + + static int PyCPointerType_SetProto(ctypes_state *st, StgInfo *stginfo, PyObject *proto) { @@ -1136,7 +1210,7 @@ PyCPointerType_init(PyObject *self, PyObject *args, PyObject *kwds) stginfo items size, align, length contain info about pointers itself, stginfo->proto has info about the pointed to type! */ - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); if (!stginfo) { return -1; @@ -1186,15 +1260,25 @@ PyCPointerType_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } +/*[clinic input] +_ctypes.PyCPointerType.set_type as PyCPointerType_set_type + + self: self(type="PyTypeObject *") + cls: defining_class + type: object + / +[clinic start generated code]*/ static PyObject * -PyCPointerType_set_type(PyTypeObject *self, PyObject *type) +PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, + PyObject *type) +/*[clinic end generated code: output=51459d8f429a70ac input=67e1e8df921f123e]*/ { PyObject *attrdict = PyType_GetDict(self); if (!attrdict) { return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); StgInfo *info; if (PyStgInfo_FromType(st, (PyObject *)self, &info) < 0) { Py_DECREF(attrdict); @@ -1223,15 +1307,28 @@ PyCPointerType_set_type(PyTypeObject *self, PyObject *type) static PyObject *_byref(ctypes_state *, PyObject *); +/*[clinic input] +_ctypes.PyCPointerType.from_param as PyCPointerType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ + static PyObject * -PyCPointerType_from_param(PyObject *type, PyObject *value) +PyCPointerType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=a4b32d929aabaf64 input=6c231276e3997884]*/ { if (value == Py_None) { /* ConvParam will convert to a NULL pointer later */ return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); StgInfo *typeinfo; if (PyStgInfo_FromType(st, type, &typeinfo) < 0) { return NULL; @@ -1273,16 +1370,16 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } } - return CDataType_from_param(type, value); + return CDataType_from_param_impl(type, cls, value); } static PyMethodDef PyCPointerType_methods[] = { - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc}, - { "from_param", (PyCFunction)PyCPointerType_from_param, METH_O, from_param_doc}, - { "set_type", (PyCFunction)PyCPointerType_set_type, METH_O }, + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF + PYCPOINTERTYPE_FROM_PARAM_METHODDEF + PYCPOINTERTYPE_SET_TYPE_METHODDEF { NULL, NULL }, }; @@ -1543,7 +1640,7 @@ PyCArrayType_init(PyObject *self, PyObject *args, PyObject *kwds) goto error; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject*)self); if (!stginfo) { goto error; @@ -1641,17 +1738,47 @@ _type_ attribute. */ +/*[clinic input] +class _ctypes.PyCSimpleType "PyObject *" "clinic_state()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d5a45772668e7f49]*/ + +/*[clinic input] +class _ctypes.c_wchar_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=468de7283d622d47]*/ + +/*[clinic input] +class _ctypes.c_char_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e750865616e7dcea]*/ + +/*[clinic input] +class _ctypes.c_void_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=dd4d9646c56f43a9]*/ + static const char SIMPLE_TYPE_CHARS[] = "cbBhHiIlLdfuzZqQPXOv?g"; +/*[clinic input] +_ctypes.c_wchar_p.from_param as c_wchar_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ + static PyObject * -c_wchar_p_from_param(PyObject *type, PyObject *value) +c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=e453949a2f725a4c input=d322c7237a319607]*/ { PyObject *as_parameter; int res; if (value == Py_None) { Py_RETURN_NONE; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls->tp_base); if (PyUnicode_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("Z"); @@ -1707,7 +1834,7 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_wchar_p_from_param(type, as_parameter); + value = c_wchar_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -1717,15 +1844,25 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) return NULL; } +/*[clinic input] +_ctypes.c_char_p.from_param as c_char_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ + static PyObject * -c_char_p_from_param(PyObject *type, PyObject *value) +c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=219652ab7c174aa1 input=6cf0d1b6bb4ede11]*/ { PyObject *as_parameter; int res; if (value == Py_None) { Py_RETURN_NONE; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls->tp_base); if (PyBytes_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("z"); @@ -1781,7 +1918,7 @@ c_char_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_char_p_from_param(type, as_parameter); + value = c_char_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -1791,8 +1928,18 @@ c_char_p_from_param(PyObject *type, PyObject *value) return NULL; } +/*[clinic input] +_ctypes.c_void_p.from_param as c_void_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ + static PyObject * -c_void_p_from_param(PyObject *type, PyObject *value) +c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=984d0075b6038cc7 input=0e8b343fc19c77d4]*/ { PyObject *as_parameter; int res; @@ -1801,7 +1948,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) if (value == Py_None) { Py_RETURN_NONE; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls->tp_base); /* Should probably allow buffer interface as well */ /* int, long */ @@ -1923,7 +2070,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_void_p_from_param(type, as_parameter); + value = c_void_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); return value; } @@ -1933,9 +2080,9 @@ c_void_p_from_param(PyObject *type, PyObject *value) return NULL; } -static PyMethodDef c_void_p_method = { "from_param", c_void_p_from_param, METH_O }; -static PyMethodDef c_char_p_method = { "from_param", c_char_p_from_param, METH_O }; -static PyMethodDef c_wchar_p_method = { "from_param", c_wchar_p_from_param, METH_O }; +static PyMethodDef c_void_p_methods[] = {C_VOID_P_FROM_PARAM_METHODDEF {0}}; +static PyMethodDef c_char_p_methods[] = {C_CHAR_P_FROM_PARAM_METHODDEF {0}}; +static PyMethodDef c_wchar_p_methods[] = {C_WCHAR_P_FROM_PARAM_METHODDEF {0}}; static PyObject *CreateSwappedType(ctypes_state *st, PyTypeObject *type, PyObject *args, PyObject *kwds, @@ -2081,7 +2228,7 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) goto error; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); if (!stginfo) { goto error; @@ -2120,15 +2267,15 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) if (((PyTypeObject *)self)->tp_base == st->Simple_Type) { switch (*proto_str) { case 'z': /* c_char_p */ - ml = &c_char_p_method; + ml = c_char_p_methods; stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'Z': /* c_wchar_p */ - ml = &c_wchar_p_method; + ml = c_wchar_p_methods; stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'P': /* c_void_p */ - ml = &c_void_p_method; + ml = c_void_p_methods; stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 's': @@ -2202,8 +2349,22 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) * This is a *class method*. * Convert a parameter into something that ConvParam can handle. */ + +/*[clinic input] +_ctypes.PyCSimpleType.from_param as PyCSimpleType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ + static PyObject * -PyCSimpleType_from_param(PyObject *type, PyObject *value) +PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=8a8453d9663e3a2e input=61cc48ce3a87a570]*/ { const char *fmt; PyCArgObject *parg; @@ -2220,7 +2381,7 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); StgInfo *info; if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; @@ -2260,7 +2421,7 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) Py_XDECREF(exc); return NULL; } - value = PyCSimpleType_from_param(type, as_parameter); + value = PyCSimpleType_from_param_impl(type, cls, as_parameter); _Py_LeaveRecursiveCall(); Py_DECREF(as_parameter); Py_XDECREF(exc); @@ -2276,11 +2437,11 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) } static PyMethodDef PyCSimpleType_methods[] = { - { "from_param", PyCSimpleType_from_param, METH_O, from_param_doc }, - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc}, + PYCSIMPLETYPE_FROM_PARAM_METHODDEF + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF { NULL, NULL }, }; @@ -2505,7 +2666,7 @@ PyCFuncPtrType_init(PyObject *self, PyObject *args, PyObject *kwds) return -1; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); if (!stginfo) { Py_DECREF(attrdict); @@ -2659,6 +2820,13 @@ KeepRef(CDataObject *target, Py_ssize_t index, PyObject *keep) /* PyCData_Type */ + +/*[clinic input] +class _ctypes.PyCData "PyObject *" "clinic_state()->PyCData_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ac13df38dee3c22c]*/ + + static int PyCData_traverse(CDataObject *self, visitproc visit, void *arg) { @@ -2731,7 +2899,7 @@ PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); StgInfo *info; if (PyStgInfo_FromObject(st, myself, &info) < 0) { return -1; @@ -2776,12 +2944,21 @@ PyCData_nohash(PyObject *self) return -1; } +/*[clinic input] +_ctypes.PyCData.__reduce__ as PyCData_reduce + + myself: self + cls: defining_class + / +[clinic start generated code]*/ + static PyObject * -PyCData_reduce(PyObject *myself, PyObject *args) +PyCData_reduce_impl(PyObject *myself, PyTypeObject *cls) +/*[clinic end generated code: output=1a025ccfdd8c935d input=34097a5226ea63c1]*/ { CDataObject *self = (CDataObject *)myself; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); StgInfo *info; if (PyStgInfo_FromObject(st, myself, &info) < 0) { return NULL; @@ -2846,7 +3023,7 @@ PyCData_from_outparam(PyObject *self, PyObject *args) static PyMethodDef PyCData_methods[] = { { "__ctypes_from_outparam__", PyCData_from_outparam, METH_NOARGS, }, - { "__reduce__", PyCData_reduce, METH_NOARGS, }, + PYCDATA_REDUCE_METHODDEF { "__setstate__", PyCData_setstate, METH_VARARGS, }, { NULL, NULL }, }; @@ -3161,7 +3338,7 @@ PyCData_set(ctypes_state *st, static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); return generic_pycdata_new(st, type, args, kwds); } @@ -3236,7 +3413,7 @@ PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ign Py_XDECREF(oldchecker); return 0; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *info; if (PyStgInfo_FromType(st, ob, &info) < 0) { return -1; @@ -3263,7 +3440,7 @@ PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) if (self->restype) { return Py_NewRef(self->restype); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *info; if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { return NULL; @@ -3285,7 +3462,7 @@ PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ig Py_CLEAR(self->converters); Py_CLEAR(self->argtypes); } else { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); converters = converters_from_argtypes(st, ob); if (!converters) return -1; @@ -3302,7 +3479,7 @@ PyCFuncPtr_get_argtypes(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) if (self->argtypes) { return Py_NewRef(self->argtypes); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *info; if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { return NULL; @@ -3351,7 +3528,7 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); StgInfo *info; if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { return NULL; @@ -3598,7 +3775,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #endif - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); return NULL; @@ -3640,7 +3817,7 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) if (paramflags == Py_None) paramflags = NULL; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { return NULL; } @@ -3721,7 +3898,7 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } */ - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); StgInfo *info; if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { return NULL; @@ -4088,7 +4265,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) int outmask; unsigned int numretvals; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *info; if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { return NULL; @@ -4312,7 +4489,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, PyObject *fields; Py_ssize_t i; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); StgInfo *baseinfo; if (PyStgInfo_FromType(st, (PyObject *)type->tp_base, &baseinfo) < 0) { return -1; @@ -4482,7 +4659,7 @@ Array_item(PyObject *myself, Py_ssize_t index) return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *stginfo; if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { return NULL; @@ -4523,7 +4700,7 @@ Array_subscript(PyObject *myself, PyObject *item) } slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *stginfo; if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { return NULL; @@ -4624,7 +4801,7 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *stginfo; if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { return -1; @@ -4815,6 +4992,12 @@ PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length) Simple_Type */ +/*[clinic input] +class _ctypes.Simple "PyObject *" "clinic_state()->Simple_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=016c476c7aa8b8a8]*/ + + static int Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) { @@ -4826,7 +5009,7 @@ Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) return -1; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *info; if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { return -1; @@ -4856,7 +5039,7 @@ Simple_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *info; if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { return NULL; @@ -4872,10 +5055,19 @@ static PyGetSetDef Simple_getsets[] = { { NULL, NULL } }; +/*[clinic input] +_ctypes.Simple.__ctypes_from_outparam__ as Simple_from_outparm + + self: self + cls: defining_class + / +[clinic start generated code]*/ + static PyObject * -Simple_from_outparm(PyObject *self, PyObject *args) +Simple_from_outparm_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=6c61d90da8aa9b4f input=0f362803fb4629d5]*/ { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); if (_ctypes_simple_instance(st, (PyObject *)Py_TYPE(self))) { return Py_NewRef(self); } @@ -4884,7 +5076,7 @@ Simple_from_outparm(PyObject *self, PyObject *args) } static PyMethodDef Simple_methods[] = { - { "__ctypes_from_outparam__", Simple_from_outparm, METH_NOARGS, }, + SIMPLE_FROM_OUTPARM_METHODDEF { NULL, NULL }, }; @@ -4898,7 +5090,7 @@ static PyObject * Simple_repr(CDataObject *self) { PyObject *val, *result; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); if (Py_TYPE(self)->tp_base != st->Simple_Type) { return PyUnicode_FromFormat("<%s object at %p>", @@ -4953,7 +5145,7 @@ Pointer_item(PyObject *myself, Py_ssize_t index) return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); StgInfo *stginfo; if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { return NULL; @@ -4997,7 +5189,7 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); StgInfo *stginfo; if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { return -1; @@ -5030,7 +5222,7 @@ Pointer_get_contents(CDataObject *self, void *closure) return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *stginfo; if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { return NULL; @@ -5053,7 +5245,7 @@ Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) "Pointer does not support item deletion"); return -1; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); StgInfo *stginfo; if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { return -1; @@ -5115,7 +5307,7 @@ Pointer_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Pointer_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); StgInfo *info; if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { return NULL; @@ -5195,7 +5387,7 @@ Pointer_subscript(PyObject *myself, PyObject *item) else len = (stop - start + 1) / step + 1; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); StgInfo *stginfo; if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { return NULL; @@ -5424,7 +5616,13 @@ cast_check_pointertype(ctypes_state *st, PyObject *arg) static PyObject * cast(void *ptr, PyObject *src, PyObject *ctype) { - ctypes_state *st = GLOBAL_STATE(); + PyObject *mod = PyType_GetModuleByDef(Py_TYPE(ctype), &_ctypesmodule); + if (!mod) { + PyErr_SetString(PyExc_TypeError, + "cast() argument 2 must be a pointer type"); + return NULL; + } + ctypes_state *st = get_module_state(mod); CDataObject *result; if (cast_check_pointertype(st, ctype) == 0) { @@ -5493,15 +5691,6 @@ wstring_at(const wchar_t *ptr, int size) } -static struct PyModuleDef _ctypesmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "_ctypes", - .m_doc = _ctypes__doc__, - .m_size = -1, - .m_methods = _ctypes_module_methods, -}; - - static int _ctypes_add_types(PyObject *mod) { @@ -5525,7 +5714,7 @@ _ctypes_add_types(PyObject *mod) } \ } while (0) - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(mod); /* Note: ob_type is the metatype (the 'type'), defaults to PyType_Type, @@ -5610,7 +5799,7 @@ _ctypes_add_objects(PyObject *mod) } \ } while (0) - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(mod); MOD_ADD("_pointer_type_cache", Py_NewRef(st->_ctypes_ptrtype_cache)); #ifdef MS_WIN32 @@ -5653,7 +5842,7 @@ _ctypes_add_objects(PyObject *mod) static int _ctypes_mod_exec(PyObject *mod) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(mod); st->_unpickle = PyObject_GetAttrString(mod, "_unpickle"); if (st->_unpickle == NULL) { return -1; @@ -5680,19 +5869,104 @@ _ctypes_mod_exec(PyObject *mod) } +static int +module_traverse(PyObject *module, visitproc visit, void *arg) { + ctypes_state *st = get_module_state(module); + Py_VISIT(st->_ctypes_ptrtype_cache); + Py_VISIT(st->_unpickle); + Py_VISIT(st->array_cache); + Py_VISIT(st->error_object_name); + Py_VISIT(st->PyExc_ArgError); + Py_VISIT(st->swapped_suffix); + + Py_VISIT(st->DictRemover_Type); + Py_VISIT(st->PyCArg_Type); + Py_VISIT(st->PyCField_Type); + Py_VISIT(st->PyCThunk_Type); + Py_VISIT(st->StructParam_Type); + Py_VISIT(st->PyCStructType_Type); + Py_VISIT(st->UnionType_Type); + Py_VISIT(st->PyCPointerType_Type); + Py_VISIT(st->PyCArrayType_Type); + Py_VISIT(st->PyCSimpleType_Type); + Py_VISIT(st->PyCFuncPtrType_Type); + Py_VISIT(st->PyCData_Type); + Py_VISIT(st->Struct_Type); + Py_VISIT(st->Union_Type); + Py_VISIT(st->PyCArray_Type); + Py_VISIT(st->Simple_Type); + Py_VISIT(st->PyCPointer_Type); + Py_VISIT(st->PyCFuncPtr_Type); +#ifdef MS_WIN32 + Py_VISIT(st->PyComError_Type); +#endif + Py_VISIT(st->PyCType_Type); + return 0; +} + +static int +module_clear(PyObject *module) { + ctypes_state *st = get_module_state(module); + Py_CLEAR(st->_ctypes_ptrtype_cache); + Py_CLEAR(st->_unpickle); + Py_CLEAR(st->array_cache); + Py_CLEAR(st->error_object_name); + Py_CLEAR(st->PyExc_ArgError); + Py_CLEAR(st->swapped_suffix); + + Py_CLEAR(st->DictRemover_Type); + Py_CLEAR(st->PyCArg_Type); + Py_CLEAR(st->PyCField_Type); + Py_CLEAR(st->PyCThunk_Type); + Py_CLEAR(st->StructParam_Type); + Py_CLEAR(st->PyCStructType_Type); + Py_CLEAR(st->UnionType_Type); + Py_CLEAR(st->PyCPointerType_Type); + Py_CLEAR(st->PyCArrayType_Type); + Py_CLEAR(st->PyCSimpleType_Type); + Py_CLEAR(st->PyCFuncPtrType_Type); + Py_CLEAR(st->PyCData_Type); + Py_CLEAR(st->Struct_Type); + Py_CLEAR(st->Union_Type); + Py_CLEAR(st->PyCArray_Type); + Py_CLEAR(st->Simple_Type); + Py_CLEAR(st->PyCPointer_Type); + Py_CLEAR(st->PyCFuncPtr_Type); +#ifdef MS_WIN32 + Py_CLEAR(st->PyComError_Type); +#endif + Py_CLEAR(st->PyCType_Type); + return 0; +} + +static void +module_free(void *module) +{ + (void)module_clear((PyObject *)module); +} + +static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _ctypes_mod_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +struct PyModuleDef _ctypesmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_ctypes", + .m_doc = _ctypes__doc__, + .m_size = sizeof(ctypes_state), + .m_methods = _ctypes_module_methods, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, +}; + PyMODINIT_FUNC PyInit__ctypes(void) { - PyObject *mod = PyModule_Create(&_ctypesmodule); - if (!mod) { - return NULL; - } - - if (_ctypes_mod_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - return mod; + return PyModuleDef_Init(&_ctypesmodule); } /* diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index b6f98e92e1ba88..7b9f6437c7d55f 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -136,6 +136,8 @@ TryAddRef(PyObject *cnv, CDataObject *obj) * Call the python object with all arguments * */ + +// BEWARE: The GIL needs to be held throughout the function static void _CallPythonObject(ctypes_state *st, void *mem, ffi_type *restype, @@ -149,7 +151,6 @@ static void _CallPythonObject(ctypes_state *st, Py_ssize_t i = 0, j = 0, nargs = 0; PyObject *error_object = NULL; int *space; - PyGILState_STATE state = PyGILState_Ensure(); assert(PyTuple_Check(converters)); nargs = PyTuple_GET_SIZE(converters); @@ -294,7 +295,6 @@ static void _CallPythonObject(ctypes_state *st, for (j = 0; j < i; j++) { Py_DECREF(args[j]); } - PyGILState_Release(state); } static void closure_fcn(ffi_cif *cif, @@ -302,8 +302,10 @@ static void closure_fcn(ffi_cif *cif, void **args, void *userdata) { + PyGILState_STATE state = PyGILState_Ensure(); + CThunkObject *p = (CThunkObject *)userdata; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(p)); _CallPythonObject(st, resp, @@ -313,6 +315,8 @@ static void closure_fcn(ffi_cif *cif, p->converters, p->flags, args); + + PyGILState_Release(state); } static CThunkObject* CThunkObject_new(ctypes_state *st, Py_ssize_t nargs) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 67d6ade43a2667..cbed2f32caa6c4 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -201,7 +201,7 @@ static PyObject * get_error_internal(PyObject *self, PyObject *args, int index) { int *space; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); PyObject *errobj = _ctypes_get_errobj(st, &space); PyObject *result; @@ -222,7 +222,7 @@ set_error_internal(PyObject *self, PyObject *args, int index) if (!PyArg_ParseTuple(args, "i", &new_errno)) { return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); errobj = _ctypes_get_errobj(st, &space); if (errobj == NULL) return NULL; @@ -1464,7 +1464,7 @@ copy_com_pointer(PyObject *self, PyObject *args) return NULL; a.keep = b.keep = NULL; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); if (ConvParam(st, p1, 0, &a) < 0 || ConvParam(st, p2, 1, &b) < 0) { goto done; } @@ -1646,7 +1646,7 @@ call_function(PyObject *self, PyObject *args) return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); result = _ctypes_callproc(st, (PPROC)func, arguments, @@ -1683,7 +1683,7 @@ call_cdeclfunction(PyObject *self, PyObject *args) return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); result = _ctypes_callproc(st, (PPROC)func, arguments, @@ -1709,7 +1709,7 @@ PyDoc_STRVAR(sizeof_doc, static PyObject * sizeof_func(PyObject *self, PyObject *obj) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); StgInfo *info; if (PyStgInfo_FromType(st, obj, &info) < 0) { @@ -1735,7 +1735,7 @@ PyDoc_STRVAR(alignment_doc, static PyObject * align_func(PyObject *self, PyObject *obj) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); StgInfo *info; if (PyStgInfo_FromAny(st, obj, &info) < 0) { return NULL; @@ -1773,7 +1773,7 @@ byref(PyObject *self, PyObject *args) if (offset == -1 && PyErr_Occurred()) return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); if (!CDataObject_Check(st, obj)) { PyErr_Format(PyExc_TypeError, "byref() argument must be a ctypes instance, not '%s'", @@ -1799,7 +1799,7 @@ PyDoc_STRVAR(addressof_doc, static PyObject * addressof(PyObject *self, PyObject *obj) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, "invalid type"); @@ -1858,7 +1858,7 @@ resize(PyObject *self, PyObject *args) &obj, &size)) return NULL; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); StgInfo *info; int result = PyStgInfo_FromObject(st, (PyObject *)obj, &info); if (result < 0) { @@ -1956,7 +1956,8 @@ create_pointer_type(PyObject *module, PyObject *cls) PyTypeObject *typ; PyObject *key; - ctypes_state *st = GLOBAL_STATE(); + assert(module); + ctypes_state *st = get_module_state(module); if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, cls, &result) != 0) { // found or error return result; @@ -2019,12 +2020,12 @@ create_pointer_inst(PyObject *module, PyObject *arg) PyObject *result; PyObject *typ; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(module); if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { return NULL; } if (typ == NULL) { - typ = create_pointer_type(NULL, (PyObject *)Py_TYPE(arg)); + typ = create_pointer_type(module, (PyObject *)Py_TYPE(arg)); if (typ == NULL) return NULL; } @@ -2039,7 +2040,7 @@ buffer_info(PyObject *self, PyObject *arg) PyObject *shape; Py_ssize_t i; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); StgInfo *info; if (PyStgInfo_FromAny(st, arg, &info) < 0) { return NULL; diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index ffe00e25aff49f..7472a4c36868a8 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -216,7 +216,7 @@ PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) { CDataObject *dst; char *ptr; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(self)); if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); @@ -240,7 +240,7 @@ PyCField_get(CFieldObject *self, PyObject *inst, PyTypeObject *type) if (inst == NULL) { return Py_NewRef(self); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(self)); if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h new file mode 100644 index 00000000000000..98a84cc14f4386 --- /dev/null +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -0,0 +1,610 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_ctypes_CType_Type___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return memory consumption of the type object."); + +#define _CTYPES_CTYPE_TYPE___SIZEOF___METHODDEF \ + {"__sizeof__", _PyCFunction_CAST(_ctypes_CType_Type___sizeof__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _ctypes_CType_Type___sizeof____doc__}, + +static PyObject * +_ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_ctypes_CType_Type___sizeof__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__sizeof__() takes no arguments"); + return NULL; + } + return _ctypes_CType_Type___sizeof___impl(self, cls); +} + +PyDoc_STRVAR(CDataType_from_address__doc__, +"from_address($self, value, /)\n" +"--\n" +"\n" +"C.from_address(integer) -> C instance\n" +"\n" +"Access a C instance at the specified address."); + +#define CDATATYPE_FROM_ADDRESS_METHODDEF \ + {"from_address", _PyCFunction_CAST(CDataType_from_address), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_address__doc__}, + +static PyObject * +CDataType_from_address_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +CDataType_from_address(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_address", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = CDataType_from_address_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_buffer__doc__, +"from_buffer($self, obj, offset=0, /)\n" +"--\n" +"\n" +"C.from_buffer(object, offset=0) -> C instance\n" +"\n" +"Create a C instance from a writeable buffer."); + +#define CDATATYPE_FROM_BUFFER_METHODDEF \ + {"from_buffer", _PyCFunction_CAST(CDataType_from_buffer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_buffer__doc__}, + +static PyObject * +CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, + Py_ssize_t offset); + +static PyObject * +CDataType_from_buffer(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_buffer", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *obj; + Py_ssize_t offset = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + obj = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } +skip_optional_posonly: + return_value = CDataType_from_buffer_impl(type, cls, obj, offset); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_buffer_copy__doc__, +"from_buffer_copy($self, buffer, offset=0, /)\n" +"--\n" +"\n" +"C.from_buffer_copy(object, offset=0) -> C instance\n" +"\n" +"Create a C instance from a readable buffer."); + +#define CDATATYPE_FROM_BUFFER_COPY_METHODDEF \ + {"from_buffer_copy", _PyCFunction_CAST(CDataType_from_buffer_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_buffer_copy__doc__}, + +static PyObject * +CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, + Py_buffer *buffer, Py_ssize_t offset); + +static PyObject * +CDataType_from_buffer_copy(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_buffer_copy", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_buffer buffer = {NULL, NULL}; + Py_ssize_t offset = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &buffer, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional_posonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } +skip_optional_posonly: + return_value = CDataType_from_buffer_copy_impl(type, cls, &buffer, offset); + +exit: + /* Cleanup for buffer */ + if (buffer.obj) { + PyBuffer_Release(&buffer); + } + + return return_value; +} + +PyDoc_STRVAR(CDataType_in_dll__doc__, +"in_dll($self, dll, name, /)\n" +"--\n" +"\n" +"C.in_dll(dll, name) -> C instance\n" +"\n" +"Access a C instance in a dll."); + +#define CDATATYPE_IN_DLL_METHODDEF \ + {"in_dll", _PyCFunction_CAST(CDataType_in_dll), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_in_dll__doc__}, + +static PyObject * +CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, + const char *name); + +static PyObject * +CDataType_in_dll(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "in_dll", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *dll; + const char *name; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + dll = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("in_dll", "argument 2", "str", args[1]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[1], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = CDataType_in_dll_impl(type, cls, dll, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define CDATATYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(CDataType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_param__doc__}, + +static PyObject * +CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +CDataType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = CDataType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCPointerType_set_type__doc__, +"set_type($self, type, /)\n" +"--\n" +"\n"); + +#define PYCPOINTERTYPE_SET_TYPE_METHODDEF \ + {"set_type", _PyCFunction_CAST(PyCPointerType_set_type), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCPointerType_set_type__doc__}, + +static PyObject * +PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, + PyObject *type); + +static PyObject * +PyCPointerType_set_type(PyTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_type", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *type; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + type = args[0]; + return_value = PyCPointerType_set_type_impl(self, cls, type); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCPointerType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define PYCPOINTERTYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(PyCPointerType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCPointerType_from_param__doc__}, + +static PyObject * +PyCPointerType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +PyCPointerType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = PyCPointerType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_wchar_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_WCHAR_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_wchar_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_wchar_p_from_param__doc__}, + +static PyObject * +c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_wchar_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_wchar_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_char_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_CHAR_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_char_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_char_p_from_param__doc__}, + +static PyObject * +c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_char_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_char_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_void_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_VOID_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_void_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_void_p_from_param__doc__}, + +static PyObject * +c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_void_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_void_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCSimpleType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define PYCSIMPLETYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(PyCSimpleType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCSimpleType_from_param__doc__}, + +static PyObject * +PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +PyCSimpleType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = PyCSimpleType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCData_reduce__doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define PYCDATA_REDUCE_METHODDEF \ + {"__reduce__", _PyCFunction_CAST(PyCData_reduce), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCData_reduce__doc__}, + +static PyObject * +PyCData_reduce_impl(PyObject *myself, PyTypeObject *cls); + +static PyObject * +PyCData_reduce(PyObject *myself, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); + return NULL; + } + return PyCData_reduce_impl(myself, cls); +} + +PyDoc_STRVAR(Simple_from_outparm__doc__, +"__ctypes_from_outparam__($self, /)\n" +"--\n" +"\n"); + +#define SIMPLE_FROM_OUTPARM_METHODDEF \ + {"__ctypes_from_outparam__", _PyCFunction_CAST(Simple_from_outparm), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Simple_from_outparm__doc__}, + +static PyObject * +Simple_from_outparm_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__ctypes_from_outparam__() takes no arguments"); + return NULL; + } + return Simple_from_outparm_impl(self, cls); +} +/*[clinic end generated code: output=9c6539a3559e6088 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 31b89dca244e8e..20c68134be2804 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -2,6 +2,9 @@ # include #endif +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() + #ifndef MS_WIN32 #define max(a, b) ((a) > (b) ? (a) : (b)) #define min(a, b) ((a) < (b) ? (a) : (b)) @@ -70,9 +73,48 @@ typedef struct { PyObject *swapped_suffix; } ctypes_state; -extern ctypes_state global_state; -#define GLOBAL_STATE() (&global_state) +extern struct PyModuleDef _ctypesmodule; + + +static inline ctypes_state * +get_module_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (ctypes_state *)state; +} + +static inline ctypes_state * +get_module_state_by_class(PyTypeObject *cls) +{ + ctypes_state *state = (ctypes_state *)_PyType_GetModuleState(cls); + assert(state != NULL); + return state; +} + +static inline ctypes_state * +get_module_state_by_def(PyTypeObject *cls) +{ + PyObject *mod = PyType_GetModuleByDef(cls, &_ctypesmodule); + assert(mod != NULL); + return get_module_state(mod); +} + +static inline ctypes_state * +get_module_state_by_def_final(PyTypeObject *cls) +{ + if (cls->tp_mro == NULL) { + return NULL; + } + PyObject *mod = PyType_GetModuleByDef(cls, &_ctypesmodule); + if (mod == NULL) { + PyErr_Clear(); + return NULL; + } + return get_module_state(mod); +} + extern PyType_Spec carg_spec; extern PyType_Spec cfield_spec; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 7b09bae0dd2a57..ad82e4891c519a 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -94,7 +94,7 @@ MakeFields(PyObject *type, CFieldObject *descr, if (fieldlist == NULL) return -1; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(descr)); PyTypeObject *cfield_tp = st->PyCField_Type; for (i = 0; i < PySequence_Fast_GET_SIZE(fieldlist); ++i) { PyObject *pair = PySequence_Fast_GET_ITEM(fieldlist, i); /* borrowed */ @@ -175,7 +175,7 @@ MakeAnonFields(PyObject *type) if (anon_names == NULL) return -1; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); PyTypeObject *cfield_tp = st->PyCField_Type; for (i = 0; i < PySequence_Fast_GET_SIZE(anon_names); ++i) { PyObject *fname = PySequence_Fast_GET_ITEM(anon_names, i); /* borrowed */ @@ -318,7 +318,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return -1; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); StgInfo *stginfo; if (PyStgInfo_FromType(st, type, &stginfo) < 0) { return -1; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 965346b9b04a32..e0ae39036c128d 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -198,6 +198,7 @@ Python/pystate.c - _Py_tss_tstate - Include/internal/pycore_blocks_output_buffer.h - BUFFER_BLOCK_SIZE - Modules/_csv.c - quote_styles - +Modules/_ctypes/_ctypes.c - _ctypesmodule - Modules/_ctypes/cfield.c - ffi_type_double - Modules/_ctypes/cfield.c - ffi_type_float - Modules/_ctypes/cfield.c - ffi_type_longdouble - From 79eec66e3dc277ea6ebad8c0b33756eea6a7ab3b Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 10 Apr 2024 10:20:05 -0400 Subject: [PATCH 139/143] gh-112536: Define `_Py_THREAD_SANITIZER` on GCC when TSan is enabled (#117702) The `__has_feature(thread_sanitizer)` is a Clang-ism. Although new versions of GCC implement `__has_feature`, the `defined(__has_feature)` check still fails on GCC so we don't use that code path. --- Include/pyport.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Include/pyport.h b/Include/pyport.h index 9d7ef0061806ad..2ba81a4be42822 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -572,6 +572,9 @@ extern "C" { # if defined(__SANITIZE_ADDRESS__) # define _Py_ADDRESS_SANITIZER # endif +# if defined(__SANITIZE_THREAD__) +# define _Py_THREAD_SANITIZER +# endif #endif From dfcae4379f2cc4d352a180f9fef2381570aa9bcb Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 10 Apr 2024 17:32:57 +0300 Subject: [PATCH 140/143] gh-115142: Skip ``test_capi.test_dict.py`` if ``_testcapi`` and ``_testlimitedcapi`` are not available (GH-117588) gh-115142: Skip test_dict if _testcapi and _testlimitedcapi is not available --- Lib/test/test_capi/test_dict.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index bcc978d224a583..e726e3d813d888 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -2,8 +2,11 @@ from collections import OrderedDict, UserDict from types import MappingProxyType from test import support -import _testcapi -import _testlimitedcapi +from test.support import import_helper + + +_testcapi = import_helper.import_module("_testcapi") +_testlimitedcapi = import_helper.import_module("_testlimitedcapi") NULL = None From 6bc0b33a91713ee62fd1860d28b19cb620c45971 Mon Sep 17 00:00:00 2001 From: Laurie O Date: Thu, 11 Apr 2024 01:01:42 +1000 Subject: [PATCH 141/143] gh-117531: Unblock getters after non-immediate queue shutdown (#117532) (This is a small tweak of the original gh-104750 which added shutdown.) --- Doc/library/queue.rst | 6 ++++-- Lib/queue.py | 8 +++++--- Lib/test/test_queue.py | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index f2a6dbf589fd87..fce23313c7de28 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -245,8 +245,10 @@ them down. queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise immediately instead. - All blocked callers of :meth:`~Queue.put` will be unblocked. If *immediate* - is true, also unblock callers of :meth:`~Queue.get` and :meth:`~Queue.join`. + All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` will be + unblocked. If *immediate* is true, a task will be marked as done for each + remaining item in the queue, which may unblock callers of + :meth:`~Queue.join`. .. versionadded:: 3.13 diff --git a/Lib/queue.py b/Lib/queue.py index 387ce5425879a4..25beb46e30d6bd 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -239,8 +239,9 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() will be unblocked, and also get() - and join() if 'immediate'. + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). ''' with self.mutex: self.is_shutdown = True @@ -249,9 +250,10 @@ def shutdown(self, immediate=False): self._get() if self.unfinished_tasks > 0: self.unfinished_tasks -= 1 - self.not_empty.notify_all() # release all blocked threads in `join()` self.all_tasks_done.notify_all() + # All getters need to re-check queue-empty to raise ShutDown + self.not_empty.notify_all() self.not_full.notify_all() # Override these methods to implement other queue organizations diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index c4d10110132393..d5927fbf39142b 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -636,6 +636,23 @@ def test_shutdown_get_task_done_join(self): self.assertEqual(results, [True]*len(thrds)) + def test_shutdown_pending_get(self): + def get(): + try: + results.append(q.get()) + except Exception as e: + results.append(e) + + q = self.type2test() + results = [] + get_thread = threading.Thread(target=get) + get_thread.start() + q.shutdown(immediate=False) + get_thread.join(timeout=10.0) + self.assertFalse(get_thread.is_alive()) + self.assertEqual(len(results), 1) + self.assertIsInstance(results[0], self.queue.ShutDown) + class QueueTest(BaseQueueTestMixin): From 630df37116b1c5b381984c547ef9d23792ceb464 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 10 Apr 2024 18:17:18 +0100 Subject: [PATCH 142/143] GH-117546: Fix symlink resolution in `os.path.realpath('loop/../link')` (#117568) Continue resolving symlink targets after encountering a symlink loop, which matches coreutils `realpath` behaviour. --- Doc/library/os.path.rst | 5 ++--- Lib/posixpath.py | 15 ++------------- Lib/test/test_posixpath.py | 2 +- ...2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst | 2 ++ 4 files changed, 7 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index fcf4b6d68e018c..ebeb3bb50b8b1f 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -409,9 +409,8 @@ the :mod:`glob` module.) style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is - resolved as far as possible and any remainder is appended without checking - whether it exists. + ``True``, :exc:`OSError` is raised. If *strict* is ``False`` these errors + are ignored, and so the result might be missing or otherwise inaccessible. .. note:: This function emulates the operating system's procedure for making a path diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 79e65587e66282..8fd49cdc358908 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -431,11 +431,6 @@ def realpath(filename, *, strict=False): # the same links. seen = {} - # Whether we're calling lstat() and readlink() to resolve symlinks. If we - # encounter an OSError for a symlink loop in non-strict mode, this is - # switched off. - querying = True - while rest: name = rest.pop() if name is None: @@ -453,9 +448,6 @@ def realpath(filename, *, strict=False): newpath = path + name else: newpath = path + sep + name - if not querying: - path = newpath - continue try: st = os.lstat(newpath) if not stat.S_ISLNK(st.st_mode): @@ -477,11 +469,8 @@ def realpath(filename, *, strict=False): if strict: # Raise OSError(errno.ELOOP) os.stat(newpath) - else: - # Return already resolved part + rest of the path unchanged. - path = newpath - querying = False - continue + path = newpath + continue seen[newpath] = None # not resolved symlink target = os.readlink(newpath) if target.startswith(sep): diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index ff78410738022d..248fe2cc5d5ca8 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -484,7 +484,7 @@ def test_realpath_symlink_loops(self): self.assertEqual(realpath(ABSTFN+"1/../x"), dirname(ABSTFN) + "/x") os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "y"), - ABSTFN + "y") + ABSTFN + "x") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "1"), ABSTFN + "1") diff --git a/Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst b/Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst new file mode 100644 index 00000000000000..9762991e47a6a4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst @@ -0,0 +1,2 @@ +Fix issue where :func:`os.path.realpath` stopped resolving symlinks after +encountering a symlink loop on POSIX. From 689ada79150f28b0053fa6c1fb646b75ab2cc200 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 10 Apr 2024 20:09:25 +0100 Subject: [PATCH 143/143] gh-67224: Make linecache imports relative to improve startup speed (#117501) --- Lib/linecache.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/linecache.py b/Lib/linecache.py index b97999fc1dc909..d1113b108dc5e4 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,9 +5,6 @@ that name. """ -import sys -import os - __all__ = ["getline", "clearcache", "checkcache", "lazycache"] @@ -66,6 +63,11 @@ def checkcache(filename=None): size, mtime, lines, fullname = entry if mtime is None: continue # no-op for files loaded via a __loader__ + try: + # This import can fail if the interpreter is shutting down + import os + except ImportError: + return try: stat = os.stat(fullname) except OSError: @@ -76,6 +78,12 @@ def checkcache(filename=None): def updatecache(filename, module_globals=None): + # These imports are not at top level because linecache is in the critical + # path of the interpreter startup and importing os and sys take a lot of time + # and slow down the startup sequence. + import os + import sys + """Update a cache entry and return its list of lines. If something's wrong, print a message, discard the cache entry, and return an empty list."""