diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index bf164d19ef2234f..b5b2765e43844f9 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -1,97 +1,16 @@
-variables:
- coverage: false
-
-trigger: ['main', '3.11', '3.10', '3.9', '3.8', '3.7']
+trigger: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7']
jobs:
- job: Prebuild
displayName: Pre-build checks
pool:
- vmImage: ubuntu-20.04
+ vmImage: ubuntu-22.04
steps:
- template: ./prebuild-checks.yml
-- job: Docs_PR
- displayName: Docs PR
- dependsOn: Prebuild
- condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
-
- pool:
- vmImage: ubuntu-20.04
-
- steps:
- - template: ./docs-steps.yml
- parameters:
- upload: true
-
-
-- job: macOS_CI_Tests
- displayName: macOS CI Tests
- dependsOn: Prebuild
- #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
- # bpo-39837: macOS tests on Azure Pipelines are disabled
- condition: false
-
- variables:
- testRunTitle: '$(build.sourceBranchName)-macos'
- testRunPlatform: macos
-
- pool:
- vmImage: macos-10.15
-
- steps:
- - template: ./macos-steps.yml
-
-
-- job: Ubuntu_CI_Tests
- displayName: Ubuntu CI Tests
- dependsOn: Prebuild
- condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
-
- pool:
- vmImage: ubuntu-20.04
-
- variables:
- testRunTitle: '$(build.sourceBranchName)-linux'
- testRunPlatform: linux
- openssl_version: 1.1.1q
-
- steps:
- - template: ./posix-steps.yml
- parameters:
- dependencies: apt
-
-
-- job: Ubuntu_Coverage_CI_Tests
- displayName: Ubuntu CI Tests (coverage)
- dependsOn: Prebuild
- condition: |
- and(
- and(
- succeeded(),
- eq(variables['coverage'], 'true')
- ),
- eq(dependencies.Prebuild.outputs['tests.run'], 'true')
- )
-
- pool:
- vmImage: ubuntu-20.04
-
- variables:
- testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
- testRunPlatform: linux-coverage
- openssl_version: 1.1.1q
-
- steps:
- - template: ./posix-steps.yml
- parameters:
- dependencies: apt
- coverage: true
-
-
- job: Windows_CI_Tests
displayName: Windows CI Tests
dependsOn: Prebuild
diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml
deleted file mode 100644
index 647daff7a033a8e..000000000000000
--- a/.azure-pipelines/docs-steps.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-parameters:
- latex: false
- upload: false
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-- task: UsePythonVersion@0
- displayName: 'Use Python 3.6 or later'
- inputs:
- versionSpec: '>=3.6'
-
-- script: python -m pip install -r requirements.txt
- workingDirectory: '$(build.sourcesDirectory)/Doc'
- displayName: 'Install build dependencies'
-
-- ${{ if ne(parameters.latex, 'true') }}:
- - script: make check html PYTHON=python
- workingDirectory: '$(build.sourcesDirectory)/Doc'
- displayName: 'Build documentation'
-
-- ${{ if eq(parameters.latex, 'true') }}:
- - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full
- displayName: 'Install LaTeX'
-
- - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'
- workingDirectory: '$(build.sourcesDirectory)/Doc'
- displayName: 'Build documentation'
-
-- ${{ if eq(parameters.upload, 'true') }}:
- - task: PublishBuildArtifacts@1
- displayName: 'Publish docs'
-
- inputs:
- PathToPublish: '$(build.sourcesDirectory)/Doc/build'
- ArtifactName: docs
- publishLocation: Container
-
- - ${{ if eq(parameters.latex, 'true') }}:
- - task: PublishBuildArtifacts@1
- displayName: 'Publish dist'
- inputs:
- PathToPublish: '$(build.sourcesDirectory)/Doc/dist'
- ArtifactName: docs_dist
- publishLocation: Container
diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml
deleted file mode 100644
index fa38a0df8c87b82..000000000000000
--- a/.azure-pipelines/macos-steps.yml
+++ /dev/null
@@ -1,27 +0,0 @@
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev
- displayName: 'Configure CPython (debug)'
-
-- script: make -j4
- displayName: 'Build CPython'
-
-- script: make pythoninfo
- displayName: 'Display build info'
-
-- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
- displayName: 'Tests'
- continueOnError: true
- timeoutInMinutes: 30
-
-- task: PublishTestResults@2
- displayName: 'Publish Test Results'
- inputs:
- testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
- mergeTestResults: true
- testRunTitle: $(testRunTitle)
- platform: $(testRunPlatform)
- condition: succeededOrFailed()
diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml
index 9d7c5e1279f46da..e23c7b1dcb55c1d 100644
--- a/.azure-pipelines/posix-steps.yml
+++ b/.azure-pipelines/posix-steps.yml
@@ -1,10 +1,3 @@
-parameters:
- coverage: false
- sudo_dependencies: sudo
- dependencies: apt
- patchcheck: true
- xvfb: true
-
steps:
- checkout: self
clean: true
@@ -14,7 +7,7 @@ steps:
- script: sudo setfacl -Rb /home/vsts
displayName: 'Workaround ACL issue'
-- script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version)
+- script: sudo ./.azure-pipelines/posix-deps-apt.sh $(openssl_version)
displayName: 'Install dependencies'
- script: ./configure --with-pydebug
@@ -23,61 +16,11 @@ steps:
- script: make -j4
displayName: 'Build CPython'
-- ${{ if eq(parameters.coverage, 'true') }}:
- - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage
- displayName: 'Set up virtual environment'
-
- - script: ./venv/bin/python -m test.pythoninfo
- displayName: 'Display build info'
-
- - script: |
- $COMMAND -m coverage run --pylib -m test \
- --fail-env-changed \
- -uall,-cpu \
- --junit-xml=$(build.binariesDirectory)/test-results.xml \
- -x test_multiprocessing_fork \
- -x test_multiprocessing_forkserver \
- -x test_multiprocessing_spawn \
- -x test_concurrent_futures
- displayName: 'Tests with coverage'
- env:
- ${{ if eq(parameters.xvfb, 'true') }}:
- COMMAND: xvfb-run ./venv/bin/python
- ${{ if ne(parameters.xvfb, 'true') }}:
- COMMAND: ./venv/bin/python
-
- - script: ./venv/bin/python -m coverage xml
- displayName: 'Generate coverage.xml'
-
- - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml
- displayName: 'Publish code coverage results'
-
-
-- ${{ if ne(parameters.coverage, 'true') }}:
- - script: make pythoninfo
- displayName: 'Display build info'
-
- - script: $COMMAND buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
- displayName: 'Tests'
- env:
- ${{ if eq(parameters.xvfb, 'true') }}:
- COMMAND: xvfb-run make
- ${{ if ne(parameters.xvfb, 'true') }}:
- COMMAND: make
-
-- ${{ if eq(parameters.patchcheck, 'true') }}:
- - script: |
- git fetch origin
- ./python Tools/patchcheck/patchcheck.py --ci true
- displayName: 'Run patchcheck.py'
- condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
-
+- script: make pythoninfo
+ displayName: 'Display build info'
-- task: PublishTestResults@2
- displayName: 'Publish Test Results'
- inputs:
- testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
- mergeTestResults: true
- testRunTitle: $(testRunTitle)
- platform: $(testRunPlatform)
- condition: succeededOrFailed()
+- script: |
+ git fetch origin
+ ./python Tools/patchcheck/patchcheck.py --ci true
+ displayName: 'Run patchcheck.py'
+ condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index 3cbd19fda982f17..335a4b407cb83cb 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -1,123 +1,28 @@
-variables:
- coverage: false
-
-pr: ['main', '3.11', '3.10', '3.9', '3.8', '3.7']
+pr: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7']
jobs:
- job: Prebuild
displayName: Pre-build checks
pool:
- vmImage: ubuntu-20.04
+ vmImage: ubuntu-22.04
steps:
- template: ./prebuild-checks.yml
-- job: Docs_PR
- displayName: Docs PR
- dependsOn: Prebuild
- condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
-
- pool:
- vmImage: ubuntu-20.04
-
- steps:
- - template: ./docs-steps.yml
-
-
-- job: macOS_PR_Tests
- displayName: macOS PR Tests
- dependsOn: Prebuild
- #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
- # bpo-39837: macOS tests on Azure Pipelines are disabled
- condition: false
-
- variables:
- testRunTitle: '$(system.pullRequest.TargetBranch)-macos'
- testRunPlatform: macos
-
- pool:
- vmImage: macos-10.15
-
- steps:
- - template: ./macos-steps.yml
- parameters:
- targetBranch: $(System.PullRequest.TargetBranch)
-
-
-- job: Ubuntu_PR_Tests
- displayName: Ubuntu PR Tests
+- job: Ubuntu_Patchcheck
+ displayName: Ubuntu patchcheck
dependsOn: Prebuild
condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
pool:
- vmImage: ubuntu-20.04
+ vmImage: ubuntu-22.04
variables:
testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
testRunPlatform: linux
- openssl_version: 1.1.1q
-
- steps:
- - template: ./posix-steps.yml
- parameters:
- dependencies: apt
-
-
-- job: Ubuntu_Coverage_PR_Tests
- displayName: Ubuntu PR Tests (coverage)
- dependsOn: Prebuild
- condition: |
- and(
- and(
- succeeded(),
- eq(variables['coverage'], 'true')
- ),
- eq(dependencies.Prebuild.outputs['tests.run'], 'true')
- )
-
- pool:
- vmImage: ubuntu-20.04
-
- variables:
- testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
- testRunPlatform: linux-coverage
- openssl_version: 1.1.1q
+ openssl_version: 1.1.1u
steps:
- template: ./posix-steps.yml
- parameters:
- dependencies: apt
- coverage: true
-
-
-- job: Windows_PR_Tests
- displayName: Windows PR Tests
- dependsOn: Prebuild
- condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
-
- pool:
- vmImage: windows-2022
-
- strategy:
- matrix:
- win32:
- arch: win32
- buildOpt: '-p Win32'
- testRunTitle: '$(System.PullRequest.TargetBranch)-win32'
- testRunPlatform: win32
- win64:
- arch: amd64
- buildOpt: '-p x64'
- testRunTitle: '$(System.PullRequest.TargetBranch)-win64'
- testRunPlatform: win64
- winarm64:
- arch: arm64
- buildOpt: '-p arm64'
- maxParallel: 4
-
- steps:
- - template: ./windows-steps.yml
- parameters:
- targetBranch: $(System.PullRequest.TargetBranch)
diff --git a/.azure-pipelines/prebuild-checks.yml b/.azure-pipelines/prebuild-checks.yml
index 30ff642d1267a1e..2c6460d2386735f 100644
--- a/.azure-pipelines/prebuild-checks.yml
+++ b/.azure-pipelines/prebuild-checks.yml
@@ -11,18 +11,6 @@ steps:
displayName: Fetch comparison tree
condition: and(succeeded(), variables['System.PullRequest.TargetBranch'])
-- script: |
- if ! git diff --name-only $(diffTarget) | grep -qE '(\.rst$|^Doc|^Misc)'
- then
- echo "No docs were updated: docs.run=false"
- echo "##vso[task.setvariable variable=run;isOutput=true]false"
- else
- echo "Docs were updated: docs.run=true"
- echo "##vso[task.setvariable variable=run;isOutput=true]true"
- fi
- displayName: Detect documentation changes
- name: docs
-
- script: |
if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)'
then
diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml
index e15729fac3443db..afd897817904947 100644
--- a/.azure-pipelines/windows-layout-steps.yml
+++ b/.azure-pipelines/windows-layout-steps.yml
@@ -12,7 +12,7 @@ steps:
displayName: Show layout info (${{ parameters.kind }})
- ${{ if eq(parameters.fulltest, 'true') }}:
- - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)"
+ - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)" -i test_launcher
workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
displayName: ${{ parameters.kind }} Tests
env:
diff --git a/.cirrus-DISABLED.yml b/.cirrus-DISABLED.yml
new file mode 100644
index 000000000000000..f20835cb6cac2af
--- /dev/null
+++ b/.cirrus-DISABLED.yml
@@ -0,0 +1,29 @@
+# gh-91960: Job disabled since Python is out of free credit (September 2023):
+# https://discuss.python.org/t/freebsd-gets-a-new-cirrus-ci-github-action-job-and-a-new-buildbot/33122/26
+
+freebsd_task:
+ freebsd_instance:
+ matrix:
+ - image: freebsd-13-2-release-amd64
+ # Turn off TCP and UDP blackhole. It is not enabled by default in FreeBSD,
+ # but it is in the FreeBSD GCE images as used by Cirrus-CI. It causes even
+ # local local connections to fail with ETIMEDOUT instead of ECONNREFUSED.
+ # For more information see https://reviews.freebsd.org/D41751 and
+ # https://github.com/cirruslabs/cirrus-ci-docs/issues/483.
+ sysctl_script:
+ - sysctl net.inet.tcp.blackhole=0
+ - sysctl net.inet.udp.blackhole=0
+ configure_script:
+ - mkdir build
+ - cd build
+ - ../configure --with-pydebug
+ build_script:
+ - cd build
+ - make -j$(sysctl -n hw.ncpu)
+ pythoninfo_script:
+ - cd build
+ - make pythoninfo
+ test_script:
+ - cd build
+ # dtrace fails to build on FreeBSD - see gh-73263
+ - make buildbottest TESTOPTS="-j0 -x test_dtrace --timeout=600"
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 000000000000000..b5d94317e8aa8b0
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,24 @@
+[run]
+branch = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+ # Don't complain if non-runnable code isn't run:
+ if 0:
+ if __name__ == .__main__.:
+ raise AssertionError\(
+
+ # Empty bodies in protocols or abstract methods
+ ^\s*def [a-zA-Z0-9_]+\(.*\)(\s*->.*)?:\s*\.\.\.(\s*#.*)?$
+ ^\s*\.\.\.(\s*#.*)?$
+
+ .*# pragma: no cover
+ .*# pragma: no branch
+
+ # Additions for IDLE:
+ .*# htest #
+ if not (_htest or _utest):
+ if not .*_utest:
+ if .*_htest:
+
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
new file mode 100644
index 000000000000000..590d7834b2b8be4
--- /dev/null
+++ b/.devcontainer/Dockerfile
@@ -0,0 +1,24 @@
+FROM docker.io/library/fedora:37
+
+ENV CC=clang
+
+ENV WASI_SDK_VERSION=20
+ENV WASI_SDK_PATH=/opt/wasi-sdk
+
+ENV WASMTIME_HOME=/opt/wasmtime
+ENV WASMTIME_VERSION=9.0.1
+ENV WASMTIME_CPU_ARCH=x86_64
+
+RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \
+ dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \
+ dnf -y clean all
+
+RUN mkdir ${WASI_SDK_PATH} && \
+ curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-linux.tar.gz | \
+ tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip
+
+RUN mkdir --parents ${WASMTIME_HOME} && \
+ curl --location "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_CPU_ARCH}-linux.tar.xz" | \
+ xz --decompress | \
+ tar --strip-components 1 --directory ${WASMTIME_HOME} -x && \
+ ln -s ${WASMTIME_HOME}/wasmtime /usr/local/bin
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 000000000000000..0dc303015df5c78
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,88 @@
+{
+ "build": {
+ "dockerfile": "Dockerfile"
+ },
+ "onCreateCommand": [
+ // Install common tooling.
+ "dnf",
+ "install",
+ "-y",
+ "which",
+ "zsh",
+ "fish",
+ // For umask fix below.
+ "/usr/bin/setfacl"
+ ],
+ "updateContentCommand": {
+ // Using the shell for `nproc` usage.
+ "python": "./configure --config-cache --with-pydebug && make -s -j `nproc`",
+ "docs": [
+ "make",
+ "--directory",
+ "Doc",
+ "venv",
+ "html"
+ ]
+ },
+ "postCreateCommand": {
+ // https://github.com/orgs/community/discussions/26026
+ "umask fix: workspace": ["sudo", "setfacl", "-bnR", "."],
+ "umask fix: /tmp": ["sudo", "setfacl", "-bnR", "/tmp"]
+ },
+ "customizations": {
+ "vscode": {
+ "extensions": [
+ // Highlighting for Parser/Python.asdl.
+ "brettcannon.zephyr-asdl",
+ // Highlighting for configure.ac.
+ "maelvalais.autoconf",
+ // C auto-complete.
+ "ms-vscode.cpptools",
+ // To view HTML build of docs.
+ "ms-vscode.live-server",
+ // Python auto-complete.
+ "ms-python.python"
+ ],
+ "settings": {
+ "C_Cpp.default.compilerPath": "/usr/bin/clang",
+ "C_Cpp.default.cStandard": "c11",
+ "C_Cpp.default.defines": [
+ "CONFIG_64",
+ "Py_BUILD_CORE"
+ ],
+ "C_Cpp.default.includePath": [
+ "${workspaceFolder}/*",
+ "${workspaceFolder}/Include/**"
+ ],
+ // https://github.com/microsoft/vscode-cpptools/issues/10732
+ "C_Cpp.errorSquiggles": "disabled",
+ "editor.insertSpaces": true,
+ "editor.rulers": [
+ 80
+ ],
+ "editor.tabSize": 4,
+ "editor.trimAutoWhitespace": true,
+ "files.associations": {
+ "*.h": "c"
+ },
+ "files.encoding": "utf8",
+ "files.eol": "\n",
+ "files.insertFinalNewline": true,
+ "files.trimTrailingWhitespace": true,
+ "python.analysis.diagnosticSeverityOverrides": {
+ // Complains about shadowing the stdlib w/ the stdlib.
+ "reportShadowedImports": "none",
+ // Doesn't like _frozen_importlib.
+ "reportMissingImports": "none"
+ },
+ "python.analysis.extraPaths": [
+ "Lib"
+ ],
+ "python.defaultInterpreterPath": "./python",
+ "[restructuredtext]": {
+ "editor.tabSize": 3
+ }
+ }
+ }
+ }
+}
diff --git a/.editorconfig b/.editorconfig
index 81445d2d79c7394..0169eed951cd3f1 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -8,5 +8,8 @@ indent_style = space
[*.{py,c,cpp,h}]
indent_size = 4
+[*.rst]
+indent_size = 3
+
[*.yml]
indent_size = 2
diff --git a/.gitattributes b/.gitattributes
index 75b33ae5c3645e0..8c37dbbb6310227 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -18,20 +18,23 @@
*.zip binary
# Specific binary files
-Lib/test/sndhdrdata/sndhdr.* binary
PC/classicAppCompat.* binary
# Text files that should not be subject to eol conversion
[attr]noeol -text
Lib/test/cjkencodings/* noeol
-Lib/test/coding20731.py noeol
+Lib/test/tokenizedata/coding20731.py noeol
Lib/test/decimaltestdata/*.decTest noeol
Lib/test/test_email/data/*.txt noeol
Lib/test/test_importlib/resources/data01/* noeol
Lib/test/test_importlib/resources/namespacedata01/* noeol
Lib/test/xmltestdata/* noeol
+# Shell scripts should have LF even on Windows because of Cygwin
+Lib/venv/scripts/common/activate text eol=lf
+Lib/venv/scripts/posix/* text eol=lf
+
# CRLF files
[attr]dos text eol=crlf
@@ -63,15 +66,20 @@ PCbuild/readme.txt dos
[attr]generated linguist-generated=true diff=generated
**/clinic/*.c.h generated
+**/clinic/*.cpp.h generated
+**/clinic/*.h.h generated
*_db.h generated
Doc/data/stable_abi.dat generated
Doc/library/token-list.inc generated
Include/internal/pycore_ast.h generated
Include/internal/pycore_ast_state.h generated
Include/internal/pycore_opcode.h generated
-Include/internal/pycore_runtime_init_generated.h generated
+Include/internal/pycore_opcode_metadata.h generated
+Include/internal/pycore_*_generated.h generated
Include/opcode.h generated
+Include/opcode_ids.h generated
Include/token.h generated
+Lib/_opcode_metadata.py generated
Lib/keyword.py generated
Lib/test/levenshtein_examples.json generated
Lib/test/test_stable_abi_ctypes.py generated
@@ -82,7 +90,9 @@ Parser/parser.c generated
Parser/token.c generated
Programs/test_frozenmain.h generated
Python/Python-ast.c generated
+Python/executor_cases.c.h generated
Python/generated_cases.c.h generated
+Python/abstract_interp_cases.c.h generated
Python/opcode_targets.h generated
Python/stdlib_module_names.h generated
Tools/peg_generator/pegen/grammar_parser.py generated
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 74081b2ef2e8af7..1925363cbeb46e1 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -5,10 +5,17 @@
# https://git-scm.com/docs/gitignore#_pattern_format
# GitHub
-.github/** @ezio-melotti
+.github/** @ezio-melotti @hugovk
+
+# pre-commit
+.pre-commit-config.yaml @hugovk @AlexWaygood
+.ruff.toml @hugovk @AlexWaygood
+
+# Build system
+configure* @erlend-aasland @corona10
# asyncio
-**/*asyncio* @1st1 @asvetlov @gvanrossum
+**/*asyncio* @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc
# Core
**/*context* @1st1
@@ -16,15 +23,23 @@
**/*hamt* @1st1
Objects/set* @rhettinger
Objects/dict* @methane @markshannon
+Objects/typevarobject.c @JelleZijlstra
Objects/type* @markshannon
Objects/codeobject.c @markshannon
Objects/frameobject.c @markshannon
Objects/call.c @markshannon
-Python/ceval.c @markshannon
+Python/ceval*.c @markshannon @gvanrossum
+Python/ceval*.h @markshannon @gvanrossum
Python/compile.c @markshannon @iritkatriel
+Python/assemble.c @markshannon @iritkatriel
+Python/flowgraph.c @markshannon @iritkatriel
Python/ast_opt.c @isidentical
+Python/bytecodes.c @markshannon @gvanrossum
+Python/optimizer*.c @markshannon @gvanrossum
Lib/test/test_patma.py @brandtbucher
Lib/test/test_peepholer.py @brandtbucher
+Lib/test/test_type_*.py @JelleZijlstra
+Lib/test/test_capi/test_misc.py @markshannon @gvanrossum
# Exceptions
Lib/traceback.py @iritkatriel
@@ -58,12 +73,9 @@ Python/traceback.c @iritkatriel
/Tools/build/parse_html5_entities.py @ezio-melotti
# Import (including importlib).
-# Ignoring importlib.h so as to not get flagged on
-# all pull requests that change the emitted
-# bytecode.
-**/*import*.c @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw
-**/*import*.py @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw
-**/*importlib/resources/* @jaraco @warsaw @brettcannon
+**/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw
+/Python/import.c @kumaraditya303
+**/*importlib/resources/* @jaraco @warsaw @FFY00
**/importlib/metadata/* @jaraco @warsaw
# Dates and times
@@ -73,7 +85,7 @@ Doc/library/time.rst @pganssle @abalkin
Lib/test/test_time.py @pganssle @abalkin
Modules/timemodule.c @pganssle @abalkin
Python/pytime.c @pganssle @abalkin
-Include/pytime.h @pganssle @abalkin
+Include/internal/pycore_time.h @pganssle @abalkin
# Email and related
**/*mail* @python/email-team
@@ -91,6 +103,11 @@ Include/pytime.h @pganssle @abalkin
/Tools/peg_generator/ @pablogsal @lysnikolaou
/Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
/Grammar/python.gram @pablogsal @lysnikolaou
+/Lib/tokenize.py @pablogsal @lysnikolaou
+/Lib/test/test_tokenize.py @pablogsal @lysnikolaou
+
+# Code generator
+/Tools/cases_generator/ @gvanrossum
# AST
Python/ast.c @isidentical
@@ -110,6 +127,12 @@ Lib/ast.py @isidentical
/Lib/test/test_subprocess.py @gpshead
/Modules/*subprocess* @gpshead
+# Limited C API & stable ABI
+Tools/build/stable_abi.py @encukou
+Misc/stable_abi.toml @encukou
+Doc/data/*.abi @encukou
+Doc/c-api/stable.rst @encukou
+
# Windows
/PC/ @python/windows-team
/PCbuild/ @python/windows-team
@@ -135,10 +158,8 @@ Lib/ast.py @isidentical
**/*idlelib* @terryjreedy
-**/*typing* @gvanrossum @Fidget-Spinner @JelleZijlstra @AlexWaygood
+**/*typing* @JelleZijlstra @AlexWaygood
-**/*asyncore @giampaolo
-**/*asynchat @giampaolo
**/*ftplib @giampaolo
**/*shutil @giampaolo
@@ -146,11 +167,26 @@ Lib/ast.py @isidentical
**/*cgi* @ethanfurman
**/*tarfile* @ethanfurman
-**/*tomllib* @encukou
+**/*tomllib* @encukou @hauntsaninja
+
+**/*sysconfig* @FFY00
+
+**/*cjkcodecs* @corona10
# macOS
/Mac/ @python/macos-team
**/*osx_support* @python/macos-team
# pathlib
-**/*pathlib* @brettcannon
+**/*pathlib* @barneygale
+
+# zipfile.Path
+**/*zipfile/_path/* @jaraco
+
+# Argument Clinic
+/Tools/clinic/** @erlend-aasland @AlexWaygood
+/Lib/test/test_clinic.py @erlend-aasland @AlexWaygood
+Doc/howto/clinic.rst @erlend-aasland
+
+# WebAssembly
+/Tools/wasm/ @brettcannon
diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md
deleted file mode 100644
index 1d93e0735e50f33..000000000000000
--- a/.github/ISSUE_TEMPLATE/bug.md
+++ /dev/null
@@ -1,32 +0,0 @@
----
-name: Bug report
-about: Submit a bug report
-labels: "type-bug"
----
-
-
-
-# Bug report
-
-A clear and concise description of what the bug is.
-Include a minimal, reproducible example (https://stackoverflow.com/help/minimal-reproducible-example), if possible.
-
-# Your environment
-
-
-
-- CPython versions tested on:
-- Operating system and architecture:
-
-
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
new file mode 100644
index 000000000000000..395516f29b037c5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug.yml
@@ -0,0 +1,56 @@
+name: Bug report
+description: Submit a bug report
+labels: ["type-bug"]
+body:
+ - type: markdown
+ attributes:
+ value: |
+ **New to Python?**
+
+ For help or advice on using Python, try one of the following options instead of opening a GitHub issue:
+
+ - Asking on [Discourse](https://discuss.python.org/c/users/7) or [Stack Overflow](https://stackoverflow.com)
+ - Reading the [Python tutorial](https://docs.python.org/3/tutorial/)
+ - Emailing [python-list](https://mail.python.org/mailman/listinfo/python-list)
+
+ Make sure to also search the [CPython issue tracker](https://github.com/python/cpython/issues?q=is%3Aissue+sort%3Acreated-desc) to check that the bug has not already been reported.
+ - type: textarea
+ attributes:
+ label: "Bug description:"
+ description: >
+ Give a clear and concise description of what happened.
+ Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible.
+ [Copy and paste code where possible rather than using screenshots](https://meta.stackoverflow.com/a/285557/13990016),
+ and put any code blocks inside triple backticks.
+
+ value: |
+ ```python
+ # Add a code block here, if required
+ ```
+ validations:
+ required: true
+ - type: dropdown
+ attributes:
+ label: "CPython versions tested on:"
+ multiple: true
+ options:
+ - "3.8"
+ - "3.9"
+ - "3.10"
+ - "3.11"
+ - "3.12"
+ - "3.13"
+ - "CPython main branch"
+ validations:
+ required: true
+ - type: dropdown
+ attributes:
+ label: "Operating systems tested on:"
+ multiple: true
+ options:
+ - Linux
+ - macOS
+ - Windows
+ - Other
+ validations:
+ required: false
diff --git a/.github/ISSUE_TEMPLATE/crash.md b/.github/ISSUE_TEMPLATE/crash.md
deleted file mode 100644
index dad3423db03410f..000000000000000
--- a/.github/ISSUE_TEMPLATE/crash.md
+++ /dev/null
@@ -1,33 +0,0 @@
----
-name: Crash report
-about: A hard crash of the interpreter, possibly with a core dump
-labels: "type-crash"
----
-
-
-
-# Crash report
-
-Tell us what happened, ideally including a minimal, reproducible example (https://stackoverflow.com/help/minimal-reproducible-example).
-
-# Error messages
-
-Enter any relevant error message caused by the crash, including a core dump if there is one.
-
-# Your environment
-
-
-
-- CPython versions tested on:
-- Operating system and architecture:
-
-
diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml
new file mode 100644
index 000000000000000..c14d7cf2599d4c1
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/crash.yml
@@ -0,0 +1,54 @@
+name: Crash report
+description: A hard crash of the interpreter, possibly with a core dump
+labels: ["type-crash"]
+body:
+ - type: markdown
+ attributes:
+ value: |
+ This form is for hard crashes of the Python interpreter, segmentation faults, failed C-level assertions, and similar. Unexpected exceptions raised from Python functions in the standard library count as bugs rather than crashes.
+
+ The CPython interpreter is written in a different programming language, C. A "CPython crash" is when Python itself fails, leading to a traceback in the C stack.
+ - type: textarea
+ attributes:
+ label: What happened?
+ description: >
+ Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible.
+ [Copy and paste code where possible rather than using screenshots](https://meta.stackoverflow.com/a/285557/13990016),
+ and put any code blocks inside triple backticks.
+
+ value: |
+ ```python
+ # Add a code block here, if required
+ ```
+ validations:
+ required: true
+ - type: dropdown
+ attributes:
+ label: "CPython versions tested on:"
+ multiple: true
+ options:
+ - "3.8"
+ - "3.9"
+ - "3.10"
+ - "3.11"
+ - "3.12"
+ - "CPython main branch"
+ validations:
+ required: true
+ - type: dropdown
+ attributes:
+ label: "Operating systems tested on:"
+ multiple: true
+ options:
+ - Linux
+ - macOS
+ - Windows
+ - Other
+ validations:
+ required: false
+ - type: input
+ attributes:
+ label: "Output from running 'python -VV' on the command line:"
+ description: If you tested with multiple operating systems or architectures, feel free to provide details in the main bug description.
+ validations:
+ required: false
diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md
deleted file mode 100644
index ed051e945f81207..000000000000000
--- a/.github/ISSUE_TEMPLATE/feature.md
+++ /dev/null
@@ -1,28 +0,0 @@
----
-name: Feature or enhancement
-about: Submit a proposal for a new CPython feature or enhancement
-labels: "type-feature"
----
-
-# Feature or enhancement
-
-(A clear and concise description of your proposal.)
-
-# Pitch
-
-(Explain why this feature or enhancement should be implemented and how it would be used.
- Add examples, if applicable.)
-
-# Previous discussion
-
-
-
-
-
diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml
new file mode 100644
index 000000000000000..4361ab2bf827fb6
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature.yml
@@ -0,0 +1,40 @@
+name: Feature or enhancement
+description: Submit a proposal for a new CPython feature or enhancement
+labels: ["type-feature"]
+body:
+ - type: markdown
+ attributes:
+ value: |
+ # Proposing a feature to CPython?
+
+ You'll need to demonstrate widespread support for your idea among the community.
+
+ Major feature proposals should generally be discussed on [Discourse](https://discuss.python.org/c/ideas/6) before opening a GitHub issue. Wait until it's clear that most people support your idea before filling in this form.
+ - type: textarea
+ attributes:
+ label: "Proposal:"
+ description: >
+ Explain your proposal, why it should be implemented, and how it would be used.
+ Add examples, if applicable.
+ Put any code blocks inside triple backticks.
+ value: |
+ ```python
+ # Add a code block here, if required
+ ```
+ validations:
+ required: true
+ - type: dropdown
+ attributes:
+ label: Has this already been discussed elsewhere?
+ options:
+ - No response given
+ - I have already discussed this feature proposal on Discourse
+ - This is a minor feature, which does not need previous discussion elsewhere
+ multiple: false
+ validations:
+ required: true
+ - type: textarea
+ attributes:
+ label: "Links to previous discussion of this feature:"
+ validations:
+ required: false
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 555e246e402bf93..c8a3165d6903645 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -12,3 +12,10 @@ updates:
update-types:
- "version-update:semver-minor"
- "version-update:semver-patch"
+ - package-ecosystem: "pip"
+ directory: "/Tools/"
+ schedule:
+ interval: "monthly"
+ labels:
+ - "skip issue"
+ - "skip news"
diff --git a/.github/problem-matchers/sphinx.json b/.github/problem-matchers/sphinx.json
deleted file mode 100644
index 09848608a7b0346..000000000000000
--- a/.github/problem-matchers/sphinx.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- "problemMatcher": [
- {
- "owner": "sphinx-problem-matcher",
- "pattern": [
- {
- "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$",
- "file": 1,
- "line": 2,
- "severity": 3,
- "message": 4
- }
- ]
- },
- {
- "owner": "sphinx-problem-matcher-loose",
- "pattern": [
- {
- "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst",
- "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$",
- "file": 1,
- "severity": 2,
- "message": 3
- }
- ]
- },
- {
- "owner": "sphinx-problem-matcher-loose-no-severity",
- "pattern": [
- {
- "_comment": "Looks for file names ending with .rst and line numbers but without severity",
- "regexp": "^(.*\\.rst):(\\d+):(.*)$",
- "file": 1,
- "line": 2,
- "message": 3
- }
- ]
- }
- ]
-}
\ No newline at end of file
diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml
new file mode 100644
index 000000000000000..1ef9178b95e5f62
--- /dev/null
+++ b/.github/workflows/add-issue-header.yml
@@ -0,0 +1,53 @@
+name: Add issue header
+# Automatically edits an issue's descriptions with a header,
+# one of:
+#
+# - Bug report
+# - Crash report
+# - Feature or enhancement
+
+on:
+ issues:
+ types:
+ # Only ever run once
+ - opened
+
+
+jobs:
+ add-header:
+ runs-on: ubuntu-latest
+ permissions:
+ issues: write
+ steps:
+ - uses: actions/github-script@v6
+ with:
+ # language=JavaScript
+ script: |
+ // https://devguide.python.org/triage/labels/#type-labels
+ const HEADERS = new Map([
+ ['type-bug', 'Bug report'],
+ ['type-crash', 'Crash report'],
+ ['type-feature', 'Feature or enhancement'],
+ ]);
+ let issue_data = await github.rest.issues.get({
+ issue_number: context.issue.number,
+ owner: context.repo.owner,
+ repo: context.repo.repo
+ }).then(issue => issue.data);
+ let header = '';
+ for (const label_data of issue_data.labels) {
+ const label_name = (typeof label_data === 'string') ? label_data : label_data.name;
+ if (HEADERS.has(label_name)) {
+ header = HEADERS.get(label_name);
+ break;
+ }
+ }
+ if (header !== '') {
+ console.log(`Setting new header: ${header}`);
+ await github.rest.issues.update({
+ issue_number: context.issue.number,
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ body: `# ${header}\n\n${issue_data.body.replaceAll('\r', '')}`
+ });
+ }
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f0544c0962e1239..cfb36c8c32e18dd 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -8,35 +8,40 @@ on:
push:
branches:
- 'main'
+ - '3.12'
- '3.11'
- '3.10'
- '3.9'
- '3.8'
- - '3.7'
pull_request:
branches:
- 'main'
+ - '3.12'
- '3.11'
- '3.10'
- '3.9'
- '3.8'
- - '3.7'
permissions:
contents: read
concurrency:
- group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable
cancel-in-progress: true
jobs:
check_source:
name: 'Check for source changes'
runs-on: ubuntu-latest
+ timeout-minutes: 10
outputs:
+ run-docs: ${{ steps.docs-changes.outputs.run-docs || false }}
run_tests: ${{ steps.check.outputs.run_tests }}
+ run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
+ run_cifuzz: ${{ steps.check.outputs.run_cifuzz }}
+ config_hash: ${{ steps.config_hash.outputs.hash }}
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Check for source changes
id: check
run: |
@@ -57,40 +62,101 @@ jobs:
# into the PR branch anyway.
#
# https://github.com/python/core-workflow/issues/373
- git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
+ git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
fi
+ # Check if we should run hypothesis tests
+ GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
+ echo $GIT_BRANCH
+ if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
+ echo "Branch too old for hypothesis tests"
+ echo "run_hypothesis=false" >> $GITHUB_OUTPUT
+ else
+ echo "Run hypothesis tests"
+ echo "run_hypothesis=true" >> $GITHUB_OUTPUT
+ fi
+
+ # oss-fuzz maintains a configuration for fuzzing the main branch of
+ # CPython, so CIFuzz should be run only for code that is likely to be
+ # merged into the main branch; compatibility with older branches may
+ # be broken.
+ FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)'
+ if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then
+ # The tests are pretty slow so they are executed only for PRs
+ # changing relevant files.
+ echo "Run CIFuzz tests"
+ echo "run_cifuzz=true" >> $GITHUB_OUTPUT
+ else
+ echo "Branch too old for CIFuzz tests; or no C files were changed"
+ echo "run_cifuzz=false" >> $GITHUB_OUTPUT
+ fi
+ - name: Compute hash for config cache key
+ id: config_hash
+ run: |
+ echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT
+ - name: Get a list of the changed documentation-related files
+ if: github.event_name == 'pull_request'
+ id: changed-docs-files
+ uses: Ana06/get-changed-files@v2.2.0
+ with:
+ filter: |
+ Doc/**
+ Misc/**
+ .github/workflows/reusable-docs.yml
+ format: csv # works for paths with spaces
+ - name: Check for docs changes
+ if: >-
+ github.event_name == 'pull_request'
+ && steps.changed-docs-files.outputs.added_modified_renamed != ''
+ id: docs-changes
+ run: |
+ echo "run-docs=true" >> "${GITHUB_OUTPUT}"
+
+ check-docs:
+ name: Docs
+ needs: check_source
+ if: fromJSON(needs.check_source.outputs.run-docs)
+ uses: ./.github/workflows/reusable-docs.yml
+
check_generated_files:
name: 'Check if generated files are up to date'
- runs-on: ubuntu-latest
+ # Don't use ubuntu-latest but a specific version to make the job
+ # reproducible: to get the same tools versions (autoconf, aclocal, ...)
+ runs-on: ubuntu-22.04
+ timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-python@v3
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '3.x'
+ - name: Restore config.cache
+ uses: actions/cache@v3
+ with:
+ path: config.cache
+ key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
- name: Install Dependencies
run: sudo ./.github/workflows/posix-deps-apt.sh
- name: Add ccache to PATH
run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- name: Configure ccache action
uses: hendrikmuhs/ccache-action@v1.2
- - name: Check Autoconf version 2.69 and aclocal 1.16.3
+ - name: Check Autoconf and aclocal versions
run: |
- grep "Generated by GNU Autoconf 2.69" configure
- grep "aclocal 1.16.3" aclocal.m4
+ grep "Generated by GNU Autoconf 2.71" configure
+ grep "aclocal 1.16.5" aclocal.m4
grep -q "runstatedir" configure
grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
- name: Configure CPython
run: |
# Build Python with the libpython dynamic library
- ./configure --with-pydebug --enable-shared
- - name: Regenerate autoconf files with container image
- run: make regen-configure
+ ./configure --config-cache --with-pydebug --enable-shared
+ - name: Regenerate autoconf files
+ # Same command used by Tools/build/regen-configure.sh ($AUTORECONF)
+ run: autoreconf -ivf -Werror
- name: Build CPython
run: |
- # Deepfreeze will usually cause global objects to be added or removed,
- # so we run it before regen-global-objects gets rum (in regen-all).
- make regen-deepfreeze
make -j4 regen-all
make regen-stdlib-module-names
- name: Check for changes
@@ -111,75 +177,90 @@ jobs:
run: make smelly
- name: Check limited ABI symbols
run: make check-limited-abi
+ - name: Check for unsupported C global variables
+ if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME
+ run: make check-c-globals
- build_win32:
- name: 'Windows (x86)'
- runs-on: windows-latest
+ build_windows:
+ name: 'Windows'
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
- env:
- IncludeUwp: 'true'
- steps:
- - uses: actions/checkout@v3
- - name: Build CPython
- run: .\PCbuild\build.bat -e -d -p Win32
- timeout-minutes: 30
- - name: Display build info
- run: .\python.bat -m test.pythoninfo
- - name: Tests
- run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+ uses: ./.github/workflows/reusable-windows.yml
- build_win_amd64:
- name: 'Windows (x64)'
- runs-on: windows-latest
+ build_windows_free_threaded:
+ name: 'Windows (free-threaded)'
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
- env:
- IncludeUwp: 'true'
- steps:
- - uses: actions/checkout@v3
- - name: Register MSVC problem matcher
- run: echo "::add-matcher::.github/problem-matchers/msvc.json"
- - name: Build CPython
- run: .\PCbuild\build.bat -e -d -p x64
- timeout-minutes: 30
- - name: Display build info
- run: .\python.bat -m test.pythoninfo
- - name: Tests
- run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+ uses: ./.github/workflows/reusable-windows.yml
+ with:
+ free-threaded: true
build_macos:
name: 'macOS'
- runs-on: macos-latest
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
- env:
- PYTHONSTRICTEXTENSIONBUILD: 1
- steps:
- - uses: actions/checkout@v3
- - name: Prepare homebrew environment variables
- run: |
- echo "LDFLAGS=-L$(brew --prefix tcl-tk)/lib" >> $GITHUB_ENV
- echo "PKG_CONFIG_PATH=$(brew --prefix openssl@1.1)/lib/pkgconfig:$(brew --prefix tcl-tk)/lib/pkgconfig" >> $GITHUB_ENV
- - name: Configure CPython
- run: ./configure --with-pydebug --prefix=/opt/python-dev
- - name: Build CPython
- run: make -j4
- - name: Display build info
- run: make pythoninfo
- - name: Tests
- run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ uses: ./.github/workflows/reusable-macos.yml
+ with:
+ config_hash: ${{ needs.check_source.outputs.config_hash }}
+
+ build_macos_free_threaded:
+ name: 'macOS (free-threaded)'
+ needs: check_source
+ if: needs.check_source.outputs.run_tests == 'true'
+ uses: ./.github/workflows/reusable-macos.yml
+ with:
+ config_hash: ${{ needs.check_source.outputs.config_hash }}
+ free-threaded: true
build_ubuntu:
name: 'Ubuntu'
+ needs: check_source
+ if: needs.check_source.outputs.run_tests == 'true'
+ uses: ./.github/workflows/reusable-ubuntu.yml
+ with:
+ config_hash: ${{ needs.check_source.outputs.config_hash }}
+ options: |
+ ../cpython-ro-srcdir/configure \
+ --config-cache \
+ --with-pydebug \
+ --with-openssl=$OPENSSL_DIR
+
+ build_ubuntu_free_threaded:
+ name: 'Ubuntu (free-threaded)'
+ needs: check_source
+ if: needs.check_source.outputs.run_tests == 'true'
+ uses: ./.github/workflows/reusable-ubuntu.yml
+ with:
+ config_hash: ${{ needs.check_source.outputs.config_hash }}
+ options: |
+ ../cpython-ro-srcdir/configure \
+ --config-cache \
+ --with-pydebug \
+ --with-openssl=$OPENSSL_DIR \
+ --disable-gil
+
+ build_ubuntu_ssltests:
+ name: 'Ubuntu SSL tests with OpenSSL'
runs-on: ubuntu-20.04
+ timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
+ strategy:
+ fail-fast: false
+ matrix:
+ openssl_ver: [1.1.1w, 3.0.11, 3.1.3]
env:
- OPENSSL_VER: 1.1.1q
- PYTHONSTRICTEXTENSIONBUILD: 1
+ OPENSSL_VER: ${{ matrix.openssl_ver }}
+ MULTISSL_DIR: ${{ github.workspace }}/multissl
+ OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}
+ LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
+ - name: Restore config.cache
+ uses: actions/cache@v3
+ with:
+ path: config.cache
+ key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
- name: Register gcc problem matcher
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Install Dependencies
@@ -203,46 +284,26 @@ jobs:
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- name: Configure ccache action
uses: hendrikmuhs/ccache-action@v1.2
- - name: Setup directory envs for out-of-tree builds
- run: |
- echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
- echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
- - name: Create directories for read-only out-of-tree builds
- run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
- - name: Bind mount sources read-only
- run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
- - name: Configure CPython out-of-tree
- working-directory: ${{ env.CPYTHON_BUILDDIR }}
- run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
- - name: Build CPython out-of-tree
- working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ - name: Configure CPython
+ run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR
+ - name: Build CPython
run: make -j4
- name: Display build info
- working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make pythoninfo
- - name: Remount sources writable for tests
- # some tests write to srcdir, lack of pyc files slows down testing
- run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
- - name: Tests
- working-directory: ${{ env.CPYTHON_BUILDDIR }}
- run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ - name: SSL tests
+ run: ./python Lib/test/ssltests.py
- build_ubuntu_ssltests:
- name: 'Ubuntu SSL tests with OpenSSL'
+ test_hypothesis:
+ name: "Hypothesis tests on Ubuntu"
runs-on: ubuntu-20.04
+ timeout-minutes: 60
needs: check_source
- if: needs.check_source.outputs.run_tests == 'true'
- strategy:
- fail-fast: false
- matrix:
- openssl_ver: [1.1.1q, 3.0.5]
+ if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
env:
- OPENSSL_VER: ${{ matrix.openssl_ver }}
- MULTISSL_DIR: ${{ github.workspace }}/multissl
- OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}
- LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
+ OPENSSL_VER: 3.0.11
+ PYTHONSTRICTEXTENSIONBUILD: 1
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Register gcc problem matcher
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Install Dependencies
@@ -266,31 +327,106 @@ jobs:
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- name: Configure ccache action
uses: hendrikmuhs/ccache-action@v1.2
- - name: Configure CPython
- run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR
- - name: Build CPython
+ - name: Setup directory envs for out-of-tree builds
+ run: |
+ echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+ echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+ - name: Create directories for read-only out-of-tree builds
+ run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+ - name: Bind mount sources read-only
+ run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+ - name: Restore config.cache
+ uses: actions/cache@v3
+ with:
+ path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
+ key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+ - name: Configure CPython out-of-tree
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ run: |
+ ../cpython-ro-srcdir/configure \
+ --config-cache \
+ --with-pydebug \
+ --with-openssl=$OPENSSL_DIR
+ - name: Build CPython out-of-tree
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make -j4
- name: Display build info
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make pythoninfo
- - name: SSL tests
- run: ./python Lib/test/ssltests.py
+ - name: Remount sources writable for tests
+ # some tests write to srcdir, lack of pyc files slows down testing
+ run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+ - name: Setup directory envs for out-of-tree builds
+ run: |
+ echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+ - name: "Create hypothesis venv"
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ run: |
+ VENV_LOC=$(realpath -m .)/hypovenv
+ VENV_PYTHON=$VENV_LOC/bin/python
+ echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
+ echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
+ ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt
+ - name: 'Restore Hypothesis database'
+ id: cache-hypothesis-database
+ uses: actions/cache@v3
+ with:
+ path: ./hypothesis
+ key: hypothesis-database-${{ github.head_ref || github.run_id }}
+ restore-keys: |
+ - hypothesis-database-
+ - name: "Run tests"
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ run: |
+ # Most of the excluded tests are slow test suites with no property tests
+ #
+ # (GH-104097) test_sysconfig is skipped because it has tests that are
+ # failing when executed from inside a virtual environment.
+ ${{ env.VENV_PYTHON }} -m test \
+ -W \
+ -o \
+ -j4 \
+ -x test_asyncio \
+ -x test_multiprocessing_fork \
+ -x test_multiprocessing_forkserver \
+ -x test_multiprocessing_spawn \
+ -x test_concurrent_futures \
+ -x test_socket \
+ -x test_subprocess \
+ -x test_signal \
+ -x test_sysconfig
+ - uses: actions/upload-artifact@v3
+ if: always()
+ with:
+ name: hypothesis-example-db
+ path: .hypothesis/examples/
build_asan:
name: 'Address sanitizer'
runs-on: ubuntu-20.04
+ timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
env:
- OPENSSL_VER: 1.1.1q
+ OPENSSL_VER: 3.0.11
PYTHONSTRICTEXTENSIONBUILD: 1
ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
+ - name: Restore config.cache
+ uses: actions/cache@v3
+ with:
+ path: config.cache
+ key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
- name: Register gcc problem matcher
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Install Dependencies
run: sudo ./.github/workflows/posix-deps-apt.sh
+ - name: Set up GCC-10 for ASAN
+ uses: egor-tensin/setup-gcc@v1
+ with:
+ version: 10
- name: Configure OpenSSL env vars
run: |
echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
@@ -311,10 +447,118 @@ jobs:
- name: Configure ccache action
uses: hendrikmuhs/ccache-action@v1.2
- name: Configure CPython
- run: ./configure --with-address-sanitizer --without-pymalloc
+ run: ./configure --config-cache --with-address-sanitizer --without-pymalloc
- name: Build CPython
run: make -j4
- name: Display build info
run: make pythoninfo
- name: Tests
- run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ run: xvfb-run make test
+
+ # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
+ cifuzz:
+ name: CIFuzz
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ needs: check_source
+ if: needs.check_source.outputs.run_cifuzz == 'true'
+ permissions:
+ security-events: write
+ strategy:
+ fail-fast: false
+ matrix:
+ sanitizer: [address, undefined, memory]
+ steps:
+ - name: Build fuzzers (${{ matrix.sanitizer }})
+ id: build
+ uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+ with:
+ oss-fuzz-project-name: cpython3
+ sanitizer: ${{ matrix.sanitizer }}
+ - name: Run fuzzers (${{ matrix.sanitizer }})
+ uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+ with:
+ fuzz-seconds: 600
+ oss-fuzz-project-name: cpython3
+ output-sarif: true
+ sanitizer: ${{ matrix.sanitizer }}
+ - name: Upload crash
+ uses: actions/upload-artifact@v3
+ if: failure() && steps.build.outcome == 'success'
+ with:
+ name: ${{ matrix.sanitizer }}-artifacts
+ path: ./out/artifacts
+ - name: Upload SARIF
+ if: always() && steps.build.outcome == 'success'
+ uses: github/codeql-action/upload-sarif@v2
+ with:
+ sarif_file: cifuzz-sarif/results.sarif
+ checkout_path: cifuzz-sarif
+
+ all-required-green: # This job does nothing and is only used for the branch protection
+ name: All required checks pass
+ if: always()
+
+ needs:
+ - check_source # Transitive dependency, needed to access `run_tests` value
+ - check-docs
+ - check_generated_files
+ - build_macos
+ - build_macos_free_threaded
+ - build_ubuntu
+ - build_ubuntu_free_threaded
+ - build_ubuntu_ssltests
+ - build_windows
+ - build_windows_free_threaded
+ - test_hypothesis
+ - build_asan
+ - cifuzz
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Check whether the needed jobs succeeded or failed
+ uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
+ with:
+ allowed-failures: >-
+ build_ubuntu_ssltests,
+ cifuzz,
+ test_hypothesis,
+ allowed-skips: >-
+ ${{
+ !fromJSON(needs.check_source.outputs.run-docs)
+ && '
+ check-docs,
+ '
+ || ''
+ }}
+ ${{
+ needs.check_source.outputs.run_tests != 'true'
+ && '
+ check_generated_files,
+ build_macos,
+ build_macos_free_threaded,
+ build_ubuntu,
+ build_ubuntu_free_threaded,
+ build_ubuntu_ssltests,
+ build_windows,
+ build_windows_free_threaded,
+ build_asan,
+ '
+ || ''
+ }}
+ ${{
+ !fromJSON(needs.check_source.outputs.run_cifuzz)
+ && '
+ cifuzz,
+ '
+ || ''
+ }}
+ ${{
+ !fromJSON(needs.check_source.outputs.run_hypothesis)
+ && '
+ test_hypothesis,
+ '
+ || ''
+ }}
+ jobs: ${{ toJSON(needs) }}
diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml
index 5f1dcae190efbc2..29282dffa37ec0d 100644
--- a/.github/workflows/build_msi.yml
+++ b/.github/workflows/build_msi.yml
@@ -8,12 +8,14 @@ on:
- '3.*'
paths:
- 'Tools/msi/**'
+ - '.github/workflows/build_msi.yml'
pull_request:
branches:
- 'main'
- '3.*'
paths:
- 'Tools/msi/**'
+ - '.github/workflows/build_msi.yml'
permissions:
contents: read
@@ -26,10 +28,11 @@ jobs:
build:
name: Windows Installer
runs-on: windows-latest
+ timeout-minutes: 60
strategy:
matrix:
type: [x86, x64, arm64]
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Build CPython installer
- run: .\Tools\msi\build.bat -${{ matrix.type }}
+ run: .\Tools\msi\build.bat --doc -${{ matrix.type }}
diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
deleted file mode 100644
index 10e4cf074a590a6..000000000000000
--- a/.github/workflows/doc.yml
+++ /dev/null
@@ -1,85 +0,0 @@
-name: Docs
-
-on:
- workflow_dispatch:
- #push:
- # branches:
- # - 'main'
- # - '3.11'
- # - '3.10'
- # - '3.9'
- # - '3.8'
- # - '3.7'
- # paths:
- # - 'Doc/**'
- pull_request:
- branches:
- - 'main'
- - '3.11'
- - '3.10'
- - '3.9'
- - '3.8'
- - '3.7'
- paths:
- - 'Doc/**'
- - 'Misc/**'
- - '.github/workflows/doc.yml'
-
-permissions:
- contents: read
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
- cancel-in-progress: true
-
-jobs:
- build_doc:
- name: 'Docs'
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - name: Register Sphinx problem matcher
- run: echo "::add-matcher::.github/problem-matchers/sphinx.json"
- - name: 'Set up Python'
- uses: actions/setup-python@v4
- with:
- python-version: '3'
- cache: 'pip'
- cache-dependency-path: 'Doc/requirements.txt'
- - name: 'Install build dependencies'
- run: make -C Doc/ venv
- - name: 'Check documentation'
- run: make -C Doc/ check
- - name: 'Build HTML documentation'
- run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html
- - name: 'Upload'
- uses: actions/upload-artifact@v3
- with:
- name: doc-html
- path: Doc/build/html
-
- # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release
- doctest:
- name: 'Doctest'
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - name: Register Sphinx problem matcher
- run: echo "::add-matcher::.github/problem-matchers/sphinx.json"
- - uses: actions/cache@v3
- with:
- path: ~/.cache/pip
- key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }}
- restore-keys: |
- ubuntu-doc-
- - name: 'Install Dependencies'
- run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican
- - name: 'Configure CPython'
- run: ./configure --with-pydebug
- - name: 'Build CPython'
- run: make -j4
- - name: 'Install build dependencies'
- run: make -C Doc/ PYTHON=../python venv
- # Use "xvfb-run" since some doctest tests open GUI windows
- - name: 'Run documentation doctest'
- run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="-W --keep-going" doctest
diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml
new file mode 100644
index 000000000000000..43a7afec73884e8
--- /dev/null
+++ b/.github/workflows/documentation-links.yml
@@ -0,0 +1,27 @@
+name: Read the Docs PR preview
+# Automatically edits a pull request's descriptions with a link
+# to the documentation's preview on Read the Docs.
+
+on:
+ pull_request_target:
+ types:
+ - opened
+ paths:
+ - 'Doc/**'
+ - '.github/workflows/doc.yml'
+
+permissions:
+ pull-requests: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ documentation-links:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: readthedocs/actions/preview@v1
+ with:
+ project-slug: "cpython-previews"
+ single-version: "true"
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
new file mode 100644
index 000000000000000..6c1c29a58cf4fc8
--- /dev/null
+++ b/.github/workflows/lint.yml
@@ -0,0 +1,26 @@
+name: Lint
+
+on: [push, pull_request, workflow_dispatch]
+
+permissions:
+ contents: read
+
+env:
+ FORCE_COLOR: 1
+ RUFF_OUTPUT_FORMAT: github
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ lint:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v4
+ with:
+ python-version: "3.x"
+ - uses: pre-commit/action@v3.0.0
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
new file mode 100644
index 000000000000000..405511ca6820b30
--- /dev/null
+++ b/.github/workflows/mypy.yml
@@ -0,0 +1,51 @@
+# Workflow to run mypy on select parts of the CPython repo
+name: mypy
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ paths:
+ - ".github/workflows/mypy.yml"
+ - "Tools/cases_generator/**"
+ - "Tools/clinic/**"
+ - "Tools/peg_generator/**"
+ - "Tools/requirements-dev.txt"
+ - "Tools/wasm/**"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+env:
+ PIP_DISABLE_PIP_VERSION_CHECK: 1
+ FORCE_COLOR: 1
+ TERM: xterm-256color # needed for FORCE_COLOR to work on mypy on Ubuntu, see https://github.com/python/mypy/issues/13817
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ mypy:
+ strategy:
+ matrix:
+ target: [
+ "Tools/cases_generator",
+ "Tools/clinic",
+ "Tools/peg_generator",
+ "Tools/wasm",
+ ]
+ name: Run mypy on ${{ matrix.target }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+ cache: pip
+ cache-dependency-path: Tools/requirements-dev.txt
+ - run: pip install -r Tools/requirements-dev.txt
+ - run: mypy --config-file ${{ matrix.target }}/mypy.ini
diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml
index b2b63472d834218..4599b21ef35f05a 100644
--- a/.github/workflows/new-bugs-announce-notifier.yml
+++ b/.github/workflows/new-bugs-announce-notifier.yml
@@ -11,21 +11,22 @@ permissions:
jobs:
notify-new-bugs-announce:
runs-on: ubuntu-latest
+ timeout-minutes: 10
steps:
- - uses: actions/setup-node@v3
+ - uses: actions/setup-node@v4
with:
- node-version: 14
+ node-version: 20
- run: npm install mailgun.js form-data
- name: Send notification
uses: actions/github-script@v6
env:
- MAILGUN_API_KEY: ${{ secrets.PSF_MAILGUN_KEY }}
+ MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }}
with:
script: |
const Mailgun = require("mailgun.js");
const formData = require('form-data');
const mailgun = new Mailgun(formData);
- const DOMAIN = "mg.python.org";
+ const DOMAIN = "mailgun.python.org";
const mg = mailgun.client({username: 'api', key: process.env.MAILGUN_API_KEY});
github.rest.issues.get({
issue_number: context.issue.number,
@@ -40,11 +41,14 @@ jobs:
url : issue.data.html_url,
labels : issue.data.labels.map(label => { return label.name }).join(", "),
assignee : issue.data.assignees.map(assignee => { return assignee.login }),
- body : issue.data.body
+ // We need to truncate the body size, because the max size for
+ // the whole payload is 16kb. We want to be safe and assume that
+ // body can take up to ~8kb of space.
+ body : issue.data.body.substring(0, 8000)
};
const data = {
- from: "CPython Issues ",
+ from: "CPython Issues ",
to: "new-bugs-announce@python.org",
subject: `[Issue ${issue.data.number}] ${issue.data.title}`,
template: "new-github-issue",
diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh
index a220896f2cd7beb..bbae378f7b994ec 100755
--- a/.github/workflows/posix-deps-apt.sh
+++ b/.github/workflows/posix-deps-apt.sh
@@ -1,9 +1,11 @@
#!/bin/sh
apt-get update
+# autoconf-archive is needed by autoreconf (check_generated_files job)
apt-get -yq install \
build-essential \
pkg-config \
+ autoconf-archive \
ccache \
gdb \
lcov \
diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml
index 99c7a05ae8cab07..7574bfc208ff768 100644
--- a/.github/workflows/project-updater.yml
+++ b/.github/workflows/project-updater.yml
@@ -13,16 +13,15 @@ jobs:
add-to-project:
name: Add issues to projects
runs-on: ubuntu-latest
+ timeout-minutes: 10
strategy:
matrix:
include:
# if an issue has any of these labels, it will be added
# to the corresponding project
- { project: 2, label: "release-blocker, deferred-blocker" }
- - { project: 3, label: expert-subinterpreters }
- - { project: 29, label: expert-asyncio }
- { project: 32, label: sprint }
-
+
steps:
- uses: actions/add-to-project@v0.1.0
with:
diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml
new file mode 100644
index 000000000000000..080204bcfd3b94f
--- /dev/null
+++ b/.github/workflows/require-pr-label.yml
@@ -0,0 +1,22 @@
+name: Check labels
+
+on:
+ pull_request:
+ types: [opened, reopened, labeled, unlabeled, synchronize]
+
+permissions:
+ issues: write
+ pull-requests: write
+
+jobs:
+ label:
+ name: DO-NOT-MERGE / unresolved review
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+
+ steps:
+ - uses: mheap/github-action-required-labels@v5
+ with:
+ mode: exactly
+ count: 0
+ labels: "DO-NOT-MERGE, awaiting changes, awaiting change review"
diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml
new file mode 100644
index 000000000000000..1c4fa4239c1e340
--- /dev/null
+++ b/.github/workflows/reusable-docs.yml
@@ -0,0 +1,108 @@
+name: Docs
+
+on:
+ workflow_call:
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ build_doc:
+ name: 'Docs'
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ branch_base: 'origin/${{ github.event.pull_request.base.ref }}'
+ branch_pr: 'origin/${{ github.event.pull_request.head.ref }}'
+ refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}'
+ refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}'
+ steps:
+ - name: 'Check out latest PR branch commit'
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
+ # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721
+ - name: 'Fetch commits to get branch diff'
+ run: |
+ # Fetch enough history to find a common ancestor commit (aka merge-base):
+ git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \
+ --no-tags --prune --no-recurse-submodules
+
+ # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from):
+ COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 ${{ env.branch_pr }} )
+ DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" )
+
+ # Get all commits since that commit date from the base branch (eg: master or main):
+ git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \
+ --no-tags --prune --no-recurse-submodules
+ - name: 'Set up Python'
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3'
+ cache: 'pip'
+ cache-dependency-path: 'Doc/requirements.txt'
+ - name: 'Install build dependencies'
+ run: make -C Doc/ venv
+
+ # To annotate PRs with Sphinx nitpicks (missing references)
+ - name: 'Build HTML documentation'
+ continue-on-error: true
+ run: |
+ set -Eeuo pipefail
+ # Build docs with the '-n' (nit-picky) option; write warnings to file
+ make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going -w sphinx-warnings.txt" html
+ - name: 'Check warnings'
+ if: github.event_name == 'pull_request'
+ run: |
+ python Doc/tools/check-warnings.py \
+ --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \
+ --fail-if-regression \
+ --fail-if-improved
+
+ # This build doesn't use problem matchers or check annotations
+ build_doc_oldest_supported_sphinx:
+ name: 'Docs (Oldest Sphinx)'
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@v4
+ - name: 'Set up Python'
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3.11' # known to work with Sphinx 4.2
+ cache: 'pip'
+ cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt'
+ - name: 'Install build dependencies'
+ run: make -C Doc/ venv REQUIREMENTS="requirements-oldest-sphinx.txt"
+ - name: 'Build HTML documentation'
+ run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html
+
+ # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release
+ doctest:
+ name: 'Doctest'
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/cache@v3
+ with:
+ path: ~/.cache/pip
+ key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }}
+ restore-keys: |
+ ubuntu-doc-
+ - name: 'Install Dependencies'
+ run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican
+ - name: 'Configure CPython'
+ run: ./configure --with-pydebug
+ - name: 'Build CPython'
+ run: make -j4
+ - name: 'Install build dependencies'
+ run: make -C Doc/ PYTHON=../python venv
+ # Use "xvfb-run" since some doctest tests open GUI windows
+ - name: 'Run documentation doctest'
+ run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="-W --keep-going" doctest
diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml
new file mode 100644
index 000000000000000..22f46d18e1b43aa
--- /dev/null
+++ b/.github/workflows/reusable-macos.yml
@@ -0,0 +1,46 @@
+on:
+ workflow_call:
+ inputs:
+ config_hash:
+ required: true
+ type: string
+ free-threaded:
+ required: false
+ type: boolean
+ default: false
+
+jobs:
+ build_macos:
+ name: 'build and test'
+ runs-on: macos-latest
+ timeout-minutes: 60
+ env:
+ HOMEBREW_NO_ANALYTICS: 1
+ HOMEBREW_NO_AUTO_UPDATE: 1
+ HOMEBREW_NO_INSTALL_CLEANUP: 1
+ PYTHONSTRICTEXTENSIONBUILD: 1
+ steps:
+ - uses: actions/checkout@v4
+ - name: Restore config.cache
+ uses: actions/cache@v3
+ with:
+ path: config.cache
+ key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }}
+ - name: Install Homebrew dependencies
+ run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk
+ - name: Configure CPython
+ run: |
+ GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
+ GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \
+ ./configure \
+ --config-cache \
+ --with-pydebug \
+ ${{ inputs.free-threaded && '--disable-gil' || '' }} \
+ --prefix=/opt/python-dev \
+ --with-openssl="$(brew --prefix openssl@3.0)"
+ - name: Build CPython
+ run: make -j4
+ - name: Display build info
+ run: make pythoninfo
+ - name: Tests
+ run: make test
diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml
new file mode 100644
index 000000000000000..819b45bda7f980a
--- /dev/null
+++ b/.github/workflows/reusable-ubuntu.yml
@@ -0,0 +1,71 @@
+on:
+ workflow_call:
+ inputs:
+ config_hash:
+ required: true
+ type: string
+ options:
+ required: true
+ type: string
+
+jobs:
+ build_ubuntu_reusable:
+ name: 'build and test'
+ timeout-minutes: 60
+ runs-on: ubuntu-20.04
+ env:
+ OPENSSL_VER: 3.0.11
+ PYTHONSTRICTEXTENSIONBUILD: 1
+ steps:
+ - uses: actions/checkout@v4
+ - name: Register gcc problem matcher
+ run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+ - name: Install dependencies
+ run: sudo ./.github/workflows/posix-deps-apt.sh
+ - name: Configure OpenSSL env vars
+ run: |
+ echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+ echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+ echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+ - name: 'Restore OpenSSL build'
+ id: cache-openssl
+ uses: actions/cache@v3
+ with:
+ path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+ key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+ - name: Install OpenSSL
+ if: steps.cache-openssl.outputs.cache-hit != 'true'
+ run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+ - name: Add ccache to PATH
+ run: |
+ echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+ - name: Configure ccache action
+ uses: hendrikmuhs/ccache-action@v1.2
+ - name: Setup directory envs for out-of-tree builds
+ run: |
+ echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+ echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+ - name: Create directories for read-only out-of-tree builds
+ run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+ - name: Bind mount sources read-only
+ run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+ - name: Restore config.cache
+ uses: actions/cache@v3
+ with:
+ path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
+ key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }}
+ - name: Configure CPython out-of-tree
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ run: ${{ inputs.options }}
+ - name: Build CPython out-of-tree
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ run: make -j4
+ - name: Display build info
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ run: make pythoninfo
+ - name: Remount sources writable for tests
+ # some tests write to srcdir, lack of pyc files slows down testing
+ run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+ - name: Tests
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ run: xvfb-run make test
diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml
new file mode 100644
index 000000000000000..29e0a7e35b54501
--- /dev/null
+++ b/.github/workflows/reusable-windows.yml
@@ -0,0 +1,53 @@
+on:
+ workflow_call:
+ inputs:
+ free-threaded:
+ required: false
+ type: boolean
+ default: false
+
+jobs:
+ build_win32:
+ name: 'build and test (x86)'
+ runs-on: windows-latest
+ timeout-minutes: 60
+ env:
+ IncludeUwp: 'true'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Build CPython
+ run: .\PCbuild\build.bat -e -d -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }}
+ - name: Display build info
+ run: .\python.bat -m test.pythoninfo
+ - name: Tests
+ run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci
+
+ build_win_amd64:
+ name: 'build and test (x64)'
+ runs-on: windows-latest
+ timeout-minutes: 60
+ env:
+ IncludeUwp: 'true'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Register MSVC problem matcher
+ run: echo "::add-matcher::.github/problem-matchers/msvc.json"
+ - name: Build CPython
+ run: .\PCbuild\build.bat -e -d -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }}
+ - name: Display build info
+ run: .\python.bat -m test.pythoninfo
+ - name: Tests
+ run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci
+
+ build_win_arm64:
+ name: 'build (arm64)'
+ runs-on: windows-latest
+ timeout-minutes: 60
+ env:
+ IncludeUwp: 'true'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Register MSVC problem matcher
+ run: echo "::add-matcher::.github/problem-matchers/msvc.json"
+ - name: Build CPython
+ run: .\PCbuild\build.bat -e -d -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }}
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 1c6f1641c885c6d..94676f5ee5fffcc 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -12,10 +12,11 @@ jobs:
if: github.repository_owner == 'python'
runs-on: ubuntu-latest
+ timeout-minutes: 10
steps:
- name: "Check PRs"
- uses: actions/stale@v6
+ uses: actions/stale@v8
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.'
diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml
index 969515ed287b55c..4a545037bf6e2bc 100644
--- a/.github/workflows/verify-ensurepip-wheels.yml
+++ b/.github/workflows/verify-ensurepip-wheels.yml
@@ -1,4 +1,4 @@
-name: Verify bundled pip and setuptools
+name: Verify bundled wheels
on:
workflow_dispatch:
@@ -23,10 +23,11 @@ concurrency:
jobs:
verify:
runs-on: ubuntu-latest
+ timeout-minutes: 10
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: '3'
- - name: Compare checksums of bundled pip and setuptools to ones published on PyPI
+ - name: Compare checksum of bundled wheels to the ones published on PyPI
run: ./Tools/build/verify_ensurepip_wheels.py
diff --git a/.gitignore b/.gitignore
index 5055e6d225c7969..8c8273fc7a3aa38 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,6 +23,10 @@
*.gc??
*.profclang?
*.profraw
+# Copies of binaries before BOLT optimizations.
+*.prebolt
+# BOLT profile data.
+*.fdata
*.dyn
.gdb_history
.purify
@@ -38,10 +42,10 @@ gmon.out
.coverage
.mypy_cache/
.pytest_cache/
+.ruff_cache/
.DS_Store
*.exe
-!Lib/distutils/command/*.exe
# Ignore core dumps... but not Tools/msi/core/ or the like.
core
@@ -58,8 +62,6 @@ Doc/.venv/
Doc/env/
Doc/.env/
Include/pydtrace_probes.h
-Lib/distutils/command/*.pdb
-Lib/lib2to3/*.pickle
Lib/site-packages/*
!Lib/site-packages/README.txt
Lib/test/data/*
@@ -126,6 +128,7 @@ Tools/unicode/data/
/platform
/profile-clean-stamp
/profile-run-stamp
+/profile-bolt-stamp
/Python/deepfreeze/*.c
/pybuilddir.txt
/pyconfig.h
diff --git a/.mailmap b/.mailmap
new file mode 100644
index 000000000000000..013c839ed6b7a4b
--- /dev/null
+++ b/.mailmap
@@ -0,0 +1,3 @@
+# This file sets the canonical name for contributors to the repository.
+# Documentation: https://git-scm.com/docs/gitmailmap
+Amethyst Reese
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000000000..35d9c64a8c5c150
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,36 @@
+repos:
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.1.2
+ hooks:
+ - id: ruff
+ name: Run Ruff on Lib/test/
+ args: [--exit-non-zero-on-fix]
+ files: ^Lib/test/
+ - id: ruff
+ name: Run Ruff on Argument Clinic
+ args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
+ files: ^Tools/clinic/|Lib/test/test_clinic.py
+
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.5.0
+ hooks:
+ - id: check-toml
+ exclude: ^Lib/test/test_tomllib/
+ - id: check-yaml
+ - id: end-of-file-fixer
+ types: [python]
+ exclude: Lib/test/tokenizedata/coding20731.py
+ - id: trailing-whitespace
+ types_or: [c, inc, python, rst]
+
+ - repo: https://github.com/sphinx-contrib/sphinx-lint
+ rev: v0.8.1
+ hooks:
+ - id: sphinx-lint
+ args: [--enable=default-role]
+ files: ^Doc/|^Misc/NEWS.d/next/
+
+ - repo: meta
+ hooks:
+ - id: check-hooks-apply
+ - id: check-useless-excludes
diff --git a/.readthedocs.yml b/.readthedocs.yml
new file mode 100644
index 000000000000000..59830c79a404e0f
--- /dev/null
+++ b/.readthedocs.yml
@@ -0,0 +1,32 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+# Project page: https://readthedocs.org/projects/cpython-previews/
+
+version: 2
+
+sphinx:
+ configuration: Doc/conf.py
+
+build:
+ os: ubuntu-22.04
+ tools:
+ python: "3"
+
+ commands:
+ # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition
+ #
+ # Cancel building pull requests when there aren't changes in the Doc directory.
+ #
+ # If there are no changes (git diff exits with 0) we force the command to return with 183.
+ # This is a special exit code on Read the Docs that will cancel the build immediately.
+ - |
+ if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ];
+ then
+ echo "No changes to Doc/ - exiting the build.";
+ exit 183;
+ fi
+
+ - make -C Doc venv html
+ - mkdir _readthedocs
+ - mv Doc/build/html _readthedocs/html
+
diff --git a/Doc/Makefile b/Doc/Makefile
index b09a9d754fb5aab..7af56e965e1be40 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -7,30 +7,29 @@
PYTHON = python3
VENVDIR = ./venv
SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build
-SPHINXLINT = PATH=$(VENVDIR)/bin:$$PATH sphinx-lint
BLURB = PATH=$(VENVDIR)/bin:$$PATH blurb
+JOBS = auto
PAPER =
SOURCES =
DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py)
+REQUIREMENTS = requirements.txt
SPHINXERRORHANDLING = -W
# Internal variables.
PAPEROPT_a4 = -D latex_elements.papersize=a4paper
PAPEROPT_letter = -D latex_elements.papersize=letterpaper
-ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j auto \
+ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j $(JOBS) \
$(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES)
-.PHONY: help build html htmlhelp latex text texinfo epub changes linkcheck \
- coverage doctest pydoc-topics htmlview clean clean-venv venv dist check serve \
- autobuild-dev autobuild-dev-html autobuild-stable autobuild-stable-html
-
+.PHONY: help
help:
@echo "Please use \`make ' where is one of"
@echo " clean to remove build files"
@echo " venv to create a venv with necessary tools"
@echo " html to make standalone HTML files"
@echo " htmlview to open the index page built by the html target in your browser"
+ @echo " htmllive to rebuild and reload HTML files in your browser"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " text to make plain text files"
@@ -44,6 +43,7 @@ help:
@echo " dist to create a \"dist\" directory with archived docs for download"
@echo " check to run a check for frequent markup errors"
+.PHONY: build
build:
-mkdir -p build
# Look first for a Misc/NEWS file (building from a source release tarball
@@ -70,38 +70,46 @@ build:
$(SPHINXBUILD) $(ALLSPHINXOPTS)
@echo
+.PHONY: html
html: BUILDER = html
html: build
@echo "Build finished. The HTML pages are in build/html."
+.PHONY: htmlhelp
htmlhelp: BUILDER = htmlhelp
htmlhelp: build
@echo "Build finished; now you can run HTML Help Workshop with the" \
"build/htmlhelp/pydoc.hhp project file."
+.PHONY: latex
latex: BUILDER = latex
latex: build
@echo "Build finished; the LaTeX files are in build/latex."
@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
"run these through (pdf)latex."
+.PHONY: text
text: BUILDER = text
text: build
@echo "Build finished; the text files are in build/text."
+.PHONY: texinfo
texinfo: BUILDER = texinfo
texinfo: build
@echo "Build finished; the python.texi file is in build/texinfo."
@echo "Run \`make info' in that directory to run it through makeinfo."
+.PHONY: epub
epub: BUILDER = epub
epub: build
@echo "Build finished; the epub files are in build/epub."
+.PHONY: changes
changes: BUILDER = changes
changes: build
@echo "The overview file is in build/changes."
+.PHONY: linkcheck
linkcheck: BUILDER = linkcheck
linkcheck:
@$(MAKE) build BUILDER=$(BUILDER) || { \
@@ -109,10 +117,12 @@ linkcheck:
"or in build/$(BUILDER)/output.txt"; \
false; }
+.PHONY: coverage
coverage: BUILDER = coverage
coverage: build
@echo "Coverage finished; see c.txt and python.txt in build/coverage"
+.PHONY: doctest
doctest: BUILDER = doctest
doctest:
@$(MAKE) build BUILDER=$(BUILDER) || { \
@@ -120,31 +130,42 @@ doctest:
"results in build/doctest/output.txt"; \
false; }
+.PHONY: pydoc-topics
pydoc-topics: BUILDER = pydoc-topics
pydoc-topics: build
@echo "Building finished; now run this:" \
"cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py"
+.PHONY: htmlview
htmlview: html
$(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))"
+.PHONY: htmllive
+htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild
+htmllive: SPHINXOPTS = --re-ignore="/venv/"
+htmllive: html
+
+.PHONY: clean
clean: clean-venv
-rm -rf build/*
+.PHONY: clean-venv
clean-venv:
rm -rf $(VENVDIR)
+.PHONY: venv
venv:
@if [ -d $(VENVDIR) ] ; then \
echo "venv already exists."; \
echo "To recreate it, remove it first with \`make clean-venv'."; \
else \
$(PYTHON) -m venv $(VENVDIR); \
- $(VENVDIR)/bin/python3 -m pip install -U pip setuptools; \
- $(VENVDIR)/bin/python3 -m pip install -r requirements.txt; \
+ $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \
+ $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \
echo "The venv has been created in the $(VENVDIR) directory"; \
fi
+.PHONY: dist
dist:
rm -rf dist
mkdir -p dist
@@ -199,12 +220,12 @@ dist:
rm -r dist/python-$(DISTVERSION)-docs-texinfo
rm dist/python-$(DISTVERSION)-docs-texinfo.tar
-check:
- # Check the docs and NEWS files with sphinx-lint.
- # Ignore the tools and venv dirs and check that the default role is not used.
- $(SPHINXLINT) -i tools -i $(VENVDIR) --enable default-role
- $(SPHINXLINT) --enable default-role ../Misc/NEWS.d/next/
+.PHONY: check
+check: venv
+ $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit
+ $(VENVDIR)/bin/python3 -m pre_commit run --all-files
+.PHONY: serve
serve:
@echo "The serve target was removed, use htmlview instead (see bpo-36329)"
@@ -216,15 +237,18 @@ serve:
# output files)
# for development releases: always build
+.PHONY: autobuild-dev
autobuild-dev:
make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
# for quick rebuilds (HTML only)
+.PHONY: autobuild-dev-html
autobuild-dev-html:
make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
# for stable releases: only build if not in pre-release stage (alpha, beta)
# release candidate downloads are okay, since the stable tree can be in that stage
+.PHONY: autobuild-stable
autobuild-stable:
@case $(DISTVERSION) in *[ab]*) \
echo "Not building; $(DISTVERSION) is not a release version."; \
@@ -232,6 +256,7 @@ autobuild-stable:
esac
@make autobuild-dev
+.PHONY: autobuild-stable-html
autobuild-stable-html:
@case $(DISTVERSION) in *[ab]*) \
echo "Not building; $(DISTVERSION) is not a release version."; \
diff --git a/Doc/_static/og-image.png b/Doc/_static/og-image.png
new file mode 100644
index 000000000000000..0e80751e740387f
Binary files /dev/null and b/Doc/_static/og-image.png differ
diff --git a/Doc/bugs.rst b/Doc/bugs.rst
index 69d7c27410d56ae..908987cf41ff6e4 100644
--- a/Doc/bugs.rst
+++ b/Doc/bugs.rst
@@ -19,6 +19,9 @@ If you find a bug in this documentation or would like to propose an improvement,
please submit a bug report on the :ref:`tracker `. If you
have a suggestion on how to fix it, include that as well.
+You can also open a discussion item on our
+`Documentation Discourse forum `_.
+
If you're short on time, you can also email documentation bug reports to
docs@python.org (behavioral bugs can be sent to python-list@python.org).
'docs@' is a mailing list run by volunteers; your request will be noticed,
@@ -35,7 +38,7 @@ though it may take a while to be processed.
`Helping with Documentation `_
Comprehensive guide for individuals that are interested in contributing to Python documentation.
- `Documentation Translations `_
+ `Documentation Translations `_
A list of GitHub pages for documentation translation and their primary contacts.
@@ -67,7 +70,7 @@ Click on the "New issue" button in the top bar to report a new issue.
The submission form has two fields, "Title" and "Comment".
For the "Title" field, enter a *very* short description of the problem;
-less than ten words is good.
+fewer than ten words is good.
In the "Comment" field, describe the problem in detail, including what you
expected to happen and what did happen. Be sure to include whether any
diff --git a/Doc/c-api/abstract.rst b/Doc/c-api/abstract.rst
index 1823f9d70c79f3e..f5df09fa7fd7863 100644
--- a/Doc/c-api/abstract.rst
+++ b/Doc/c-api/abstract.rst
@@ -24,4 +24,3 @@ but whose items have not been set to some non-\ ``NULL`` value yet.
mapping.rst
iter.rst
buffer.rst
- objbuffer.rst
diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst
index 0a8fcc5ae5fcdf6..b3609c233156b6e 100644
--- a/Doc/c-api/allocation.rst
+++ b/Doc/c-api/allocation.rst
@@ -27,22 +27,26 @@ Allocating Objects on the Heap
length information for a variable-size object.
-.. c:function:: TYPE* PyObject_New(TYPE, PyTypeObject *type)
+.. c:macro:: PyObject_New(TYPE, typeobj)
- Allocate a new Python object using the C structure type *TYPE* and the
- Python type object *type*. Fields not defined by the Python object header
- are not initialized; the object's reference count will be one. The size of
- the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of
- the type object.
+ Allocate a new Python object using the C structure type *TYPE*
+ and the Python type object *typeobj* (``PyTypeObject*``).
+ Fields not defined by the Python object header are not initialized.
+ The caller will own the only reference to the object
+ (i.e. its reference count will be one).
+ The size of the memory allocation is determined from the
+ :c:member:`~PyTypeObject.tp_basicsize` field of the type object.
-.. c:function:: TYPE* PyObject_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size)
+.. c:macro:: PyObject_NewVar(TYPE, typeobj, size)
Allocate a new Python object using the C structure type *TYPE* and the
- Python type object *type*. Fields not defined by the Python object header
+ Python type object *typeobj* (``PyTypeObject*``).
+ Fields not defined by the Python object header
are not initialized. The allocated memory allows for the *TYPE* structure
- plus *size* fields of the size given by the :c:member:`~PyTypeObject.tp_itemsize` field of
- *type*. This is useful for implementing objects like tuples, which are
+ plus *size* (``Py_ssize_t``) fields of the size
+ given by the :c:member:`~PyTypeObject.tp_itemsize` field of
+ *typeobj*. This is useful for implementing objects like tuples, which are
able to determine their size at construction time. Embedding the array of
fields into the same allocation decreases the number of allocations,
improving the memory management efficiency.
@@ -50,8 +54,8 @@ Allocating Objects on the Heap
.. c:function:: void PyObject_Del(void *op)
- Releases memory allocated to an object using :c:func:`PyObject_New` or
- :c:func:`PyObject_NewVar`. This is normally called from the
+ Releases memory allocated to an object using :c:macro:`PyObject_New` or
+ :c:macro:`PyObject_NewVar`. This is normally called from the
:c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of
the object should not be accessed after this call as the memory is no
longer a valid Python object.
diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst
index 85b6e2f373877f2..f6c8284daeacb0a 100644
--- a/Doc/c-api/apiabiversion.rst
+++ b/Doc/c-api/apiabiversion.rst
@@ -58,7 +58,9 @@ See :ref:`stable` for a discussion of API and ABI stability across versions.
Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is
hexversion ``0x030a00f0``.
- This version is also available via the symbol :data:`Py_Version`.
+ Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``.
+
+ This version is also available via the symbol :c:var:`Py_Version`.
.. c:var:: const unsigned long Py_Version
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst
index c5be453c153308f..62d87d898e682c5 100644
--- a/Doc/c-api/arg.rst
+++ b/Doc/c-api/arg.rst
@@ -27,40 +27,55 @@ unit; the entry in (round) parentheses is the Python object type that matches
the format unit; and the entry in [square] brackets is the type of the C
variable(s) whose address should be passed.
+.. _arg-parsing-string-and-buffers:
+
Strings and buffers
-------------------
+.. note::
+
+ On Python 3.12 and older, the macro :c:macro:`!PY_SSIZE_T_CLEAN` must be
+ defined before including :file:`Python.h` to use all ``#`` variants of
+ formats (``s#``, ``y#``, etc.) explained below.
+ This is not necessary on Python 3.13 and later.
+
These formats allow accessing an object as a contiguous chunk of memory.
You don't have to provide raw storage for the returned unicode or bytes
area.
-In general, when a format sets a pointer to a buffer, the buffer is
-managed by the corresponding Python object, and the buffer shares
-the lifetime of this object. You won't have to release any memory yourself.
-The only exceptions are ``es``, ``es#``, ``et`` and ``et#``.
+Unless otherwise stated, buffers are not NUL-terminated.
-However, when a :c:type:`Py_buffer` structure gets filled, the underlying
-buffer is locked so that the caller can subsequently use the buffer even
-inside a :c:type:`Py_BEGIN_ALLOW_THREADS` block without the risk of mutable data
-being resized or destroyed. As a result, **you have to call**
-:c:func:`PyBuffer_Release` after you have finished processing the data (or
-in any early abort case).
+There are three ways strings and buffers can be converted to C:
-Unless otherwise stated, buffers are not NUL-terminated.
+* Formats such as ``y*`` and ``s*`` fill a :c:type:`Py_buffer` structure.
+ This locks the underlying buffer so that the caller can subsequently use
+ the buffer even inside a :c:type:`Py_BEGIN_ALLOW_THREADS`
+ block without the risk of mutable data being resized or destroyed.
+ As a result, **you have to call** :c:func:`PyBuffer_Release` after you have
+ finished processing the data (or in any early abort case).
-Some formats require a read-only :term:`bytes-like object`, and set a
-pointer instead of a buffer structure. They work by checking that
-the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is ``NULL``,
-which disallows mutable objects such as :class:`bytearray`.
+* The ``es``, ``es#``, ``et`` and ``et#`` formats allocate the result buffer.
+ **You have to call** :c:func:`PyMem_Free` after you have finished
+ processing the data (or in any early abort case).
-.. note::
+* .. _c-arg-borrowed-buffer:
+
+ Other formats take a :class:`str` or a read-only :term:`bytes-like object`,
+ such as :class:`bytes`, and provide a ``const char *`` pointer to
+ its buffer.
+ In this case the buffer is "borrowed": it is managed by the corresponding
+ Python object, and shares the lifetime of this object.
+ You won't have to release any memory yourself.
- For all ``#`` variants of formats (``s#``, ``y#``, etc.), the macro
- :c:macro:`PY_SSIZE_T_CLEAN` must be defined before including
- :file:`Python.h`. On Python 3.9 and older, the type of the length argument
- is :c:type:`Py_ssize_t` if the :c:macro:`PY_SSIZE_T_CLEAN` macro is defined,
- or int otherwise.
+ To ensure that the underlying buffer may be safely borrowed, the object's
+ :c:member:`PyBufferProcs.bf_releasebuffer` field must be ``NULL``.
+ This disallows common mutable objects such as :class:`bytearray`,
+ but also some read-only objects such as :class:`memoryview` of
+ :class:`bytes`.
+ Besides this ``bf_releasebuffer`` requirement, there is no check to verify
+ whether the input object is immutable (e.g. whether it would honor a request
+ for a writable buffer, or whether another thread can mutate the data).
``s`` (:class:`str`) [const char \*]
Convert a Unicode object to a C pointer to a character string.
@@ -89,7 +104,7 @@ which disallows mutable objects such as :class:`bytearray`.
Unicode objects are converted to C strings using ``'utf-8'`` encoding.
``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`]
- Like ``s*``, except that it doesn't accept mutable objects.
+ Like ``s*``, except that it provides a :ref:`borrowed buffer `.
The result is stored into two C variables,
the first one a pointer to a C string, the second one its length.
The string may contain embedded null bytes. Unicode objects are converted
@@ -108,8 +123,9 @@ which disallows mutable objects such as :class:`bytearray`.
pointer is set to ``NULL``.
``y`` (read-only :term:`bytes-like object`) [const char \*]
- This format converts a bytes-like object to a C pointer to a character
- string; it does not accept Unicode objects. The bytes buffer must not
+ This format converts a bytes-like object to a C pointer to a
+ :ref:`borrowed ` character string;
+ it does not accept Unicode objects. The bytes buffer must not
contain embedded null bytes; if it does, a :exc:`ValueError`
exception is raised.
@@ -277,8 +293,10 @@ Other objects
``O`` (object) [PyObject \*]
Store a Python object (without any conversion) in a C object pointer. The C
- program thus receives the actual object that was passed. The object's reference
- count is not increased. The pointer stored is not ``NULL``.
+ program thus receives the actual object that was passed. A new
+ :term:`strong reference` to the object is not created
+ (i.e. its reference count is not increased).
+ The pointer stored is not ``NULL``.
``O!`` (object) [*typeobject*, PyObject \*]
Store a Python object in a C object pointer. This is similar to ``O``, but
@@ -327,7 +345,7 @@ Other objects
*items*. Format units for sequences may be nested.
It is possible to pass "long" integers (integers whose value exceeds the
-platform's :const:`LONG_MAX`) however no proper range checking is done --- the
+platform's :c:macro:`LONG_MAX`) however no proper range checking is done --- the
most significant bits are silently truncated when the receiving field is too
small to receive the value (actually, the semantics are inherited from downcasts
in C --- your mileage may vary).
@@ -362,7 +380,8 @@ inside nested parentheses. They are:
mutually exclude each other.
Note that any Python object references which are provided to the caller are
-*borrowed* references; do not decrement their reference count!
+*borrowed* references; do not release them
+(i.e. do not decrement their reference count)!
Additional arguments passed to these functions must be addresses of variables
whose type is determined by the format string; these are used to store values
@@ -397,8 +416,10 @@ API Functions
.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...)
Parse the parameters of a function that takes both positional and keyword
- parameters into local variables. The *keywords* argument is a
- ``NULL``-terminated array of keyword parameter names. Empty names denote
+ parameters into local variables.
+ The *keywords* argument is a ``NULL``-terminated array of keyword parameter
+ names specified as null-terminated ASCII or UTF-8 encoded C strings.
+ Empty names denote
:ref:`positional-only parameters `.
Returns true on success; on failure, it returns false and raises the
appropriate exception.
@@ -407,6 +428,9 @@ API Functions
Added support for :ref:`positional-only parameters
`.
+ .. versionchanged:: 3.13
+ Added support for non-ASCII keyword parameter names.
+
.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs)
@@ -423,23 +447,31 @@ API Functions
.. versionadded:: 3.2
-.. XXX deprecated, will be removed
.. c:function:: int PyArg_Parse(PyObject *args, const char *format, ...)
- Function used to deconstruct the argument lists of "old-style" functions ---
- these are functions which use the :const:`METH_OLDARGS` parameter parsing
- method, which has been removed in Python 3. This is not recommended for use
- in parameter parsing in new code, and most code in the standard interpreter
- has been modified to no longer use this for that purpose. It does remain a
- convenient way to decompose other tuples, however, and may continue to be
- used for that purpose.
+ Parse the parameter of a function that takes a single positional parameter
+ into a local variable. Returns true on success; on failure, it returns
+ false and raises the appropriate exception.
+
+ Example::
+
+ // Function using METH_O calling convention
+ static PyObject*
+ my_function(PyObject *module, PyObject *arg)
+ {
+ int value;
+ if (!PyArg_Parse(arg, "i:my_function", &value)) {
+ return NULL;
+ }
+ // ... use value ...
+ }
.. c:function:: int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...)
A simpler form of parameter retrieval which does not use a format string to
specify the types of the arguments. Functions which use this method to retrieve
- their parameters should be declared as :const:`METH_VARARGS` in function or
+ their parameters should be declared as :c:macro:`METH_VARARGS` in function or
method tables. The tuple containing the actual parameters should be passed as
*args*; it must actually be a tuple. The length of the tuple must be at least
*min* and no more than *max*; *min* and *max* may be equal. Additional
@@ -453,7 +485,7 @@ API Functions
will be set if there was a failure.
This is an example of the use of this function, taken from the sources for the
- :mod:`_weakref` helper module for weak references::
+ :mod:`!_weakref` helper module for weak references::
static PyObject *
weakref_ref(PyObject *self, PyObject *args)
@@ -531,7 +563,7 @@ Building values
Same as ``s#``.
``u`` (:class:`str`) [const wchar_t \*]
- Convert a null-terminated :c:expr:`wchar_t` buffer of Unicode (UTF-16 or UCS-4)
+ Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4)
data to a Python Unicode object. If the Unicode buffer pointer is ``NULL``,
``None`` is returned.
@@ -597,8 +629,10 @@ Building values
Convert a C :c:type:`Py_complex` structure to a Python complex number.
``O`` (object) [PyObject \*]
- Pass a Python object untouched (except for its reference count, which is
- incremented by one). If the object passed in is a ``NULL`` pointer, it is assumed
+ Pass a Python object untouched but create a new
+ :term:`strong reference` to it
+ (i.e. its reference count is incremented by one).
+ If the object passed in is a ``NULL`` pointer, it is assumed
that this was caused because the call producing the argument found an error and
set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't
raise an exception. If no exception has been raised yet, :exc:`SystemError` is
@@ -608,7 +642,7 @@ Building values
Same as ``O``.
``N`` (object) [PyObject \*]
- Same as ``O``, except it doesn't increment the reference count on the object.
+ Same as ``O``, except it doesn't create a new :term:`strong reference`.
Useful when the object is created by a call to an object constructor in the
argument list.
diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst
index c197d447e9618c9..b4dc4849d044e1c 100644
--- a/Doc/c-api/bool.rst
+++ b/Doc/c-api/bool.rst
@@ -6,11 +6,17 @@ Boolean Objects
---------------
Booleans in Python are implemented as a subclass of integers. There are only
-two booleans, :const:`Py_False` and :const:`Py_True`. As such, the normal
+two booleans, :c:data:`Py_False` and :c:data:`Py_True`. As such, the normal
creation and deletion functions don't apply to booleans. The following macros
are available, however.
+.. c:var:: PyTypeObject PyBool_Type
+
+ This instance of :c:type:`PyTypeObject` represents the Python boolean type; it
+ is the same object as :class:`bool` in the Python layer.
+
+
.. c:function:: int PyBool_Check(PyObject *o)
Return true if *o* is of type :c:data:`PyBool_Type`. This function always
@@ -19,29 +25,32 @@ are available, however.
.. c:var:: PyObject* Py_False
- The Python ``False`` object. This object has no methods. It needs to be
- treated just like any other object with respect to reference counts.
+ The Python ``False`` object. This object has no methods and is
+ :term:`immortal`.
+
+ .. versionchanged:: 3.12
+ :c:data:`Py_False` is :term:`immortal`.
.. c:var:: PyObject* Py_True
- The Python ``True`` object. This object has no methods. It needs to be treated
- just like any other object with respect to reference counts.
+ The Python ``True`` object. This object has no methods and is
+ :term:`immortal`.
+
+ .. versionchanged:: 3.12
+ :c:data:`Py_True` is :term:`immortal`.
.. c:macro:: Py_RETURN_FALSE
- Return :const:`Py_False` from a function, properly incrementing its reference
- count.
+ Return :c:data:`Py_False` from a function.
.. c:macro:: Py_RETURN_TRUE
- Return :const:`Py_True` from a function, properly incrementing its reference
- count.
+ Return :c:data:`Py_True` from a function.
.. c:function:: PyObject* PyBool_FromLong(long v)
- Return a new reference to :const:`Py_True` or :const:`Py_False` depending on the
- truth value of *v*.
+ Return :c:data:`Py_True` or :c:data:`Py_False`, depending on the truth value of *v*.
diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index a04062fb2a68f1d..e572815ffd6259f 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -44,7 +44,7 @@ the elements exposed by an :class:`array.array` can be multi-byte values.
An example consumer of the buffer interface is the :meth:`~io.BufferedIOBase.write`
method of file objects: any object that can export a series of bytes through
-the buffer interface can be written to a file. While :meth:`write` only
+the buffer interface can be written to a file. While :meth:`!write` only
needs read-only access to the internal contents of the object passed to it,
other methods such as :meth:`~io.BufferedIOBase.readinto` need write access
to the contents of their argument. The buffer interface allows objects to
@@ -102,7 +102,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`.
.. c:member:: PyObject *obj
A new reference to the exporting object. The reference is owned by
- the consumer and automatically decremented and set to ``NULL`` by
+ the consumer and automatically released
+ (i.e. reference count decremented)
+ and set to ``NULL`` by
:c:func:`PyBuffer_Release`. The field is the equivalent of the return
value of any standard C-API function.
@@ -159,10 +161,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`.
If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing
a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides`
and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``.
-
- The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions
- to 64. Exporters MUST respect this limit, consumers of multi-dimensional
- buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions.
+ The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`.
.. c:member:: Py_ssize_t *shape
@@ -215,6 +214,17 @@ a buffer, see :c:func:`PyObject_GetBuffer`.
freed when the buffer is released. The consumer MUST NOT alter this
value.
+
+Constants:
+
+.. c:macro:: PyBUF_MAX_NDIM
+
+ The maximum number of dimensions the memory represents.
+ Exporters MUST respect this limit, consumers of multi-dimensional
+ buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions.
+ Currently set to 64.
+
+
.. _buffer-request-types:
Buffer request types
@@ -225,7 +235,7 @@ object via :c:func:`PyObject_GetBuffer`. Since the complexity of the logical
structure of the memory can vary drastically, the consumer uses the *flags*
argument to specify the exact buffer type it can handle.
-All :c:data:`Py_buffer` fields are unambiguously defined by the request
+All :c:type:`Py_buffer` fields are unambiguously defined by the request
type.
request-independent fields
@@ -438,7 +448,7 @@ Buffer-related functions
Send a request to *exporter* to fill in *view* as specified by *flags*.
If the exporter cannot provide a buffer of the exact type, it MUST raise
- :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and
+ :exc:`BufferError`, set ``view->obj`` to ``NULL`` and
return ``-1``.
On success, fill in *view*, set ``view->obj`` to a new reference
@@ -454,7 +464,8 @@ Buffer-related functions
.. c:function:: void PyBuffer_Release(Py_buffer *view)
- Release the buffer *view* and decrement the reference count for
+ Release the buffer *view* and release the :term:`strong reference`
+ (i.e. decrement the reference count) to the view's supporting object,
``view->obj``. This function MUST be called when the buffer
is no longer being used, otherwise reference leaks may occur.
@@ -464,7 +475,7 @@ Buffer-related functions
.. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *format)
- Return the implied :c:data:`~Py_buffer.itemsize` from :c:data:`~Py_buffer.format`.
+ Return the implied :c:member:`~Py_buffer.itemsize` from :c:member:`~Py_buffer.format`.
On error, raise an exception and return -1.
.. versionadded:: 3.9
@@ -499,7 +510,7 @@ Buffer-related functions
This function fails if *len* != *src->len*.
-.. c:function:: int PyObject_CopyData(Py_buffer *dest, Py_buffer *src)
+.. c:function:: int PyObject_CopyData(PyObject *dest, PyObject *src)
Copy data from *src* to *dest* buffer. Can convert between C-style and
or Fortran-style buffers.
@@ -524,7 +535,7 @@ Buffer-related functions
and :c:macro:`PyBUF_WRITABLE` is set in *flags*.
On success, set ``view->obj`` to a new reference to *exporter* and
- return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set
+ return 0. Otherwise, raise :exc:`BufferError`, set
``view->obj`` to ``NULL`` and return ``-1``;
If this function is used as part of a :ref:`getbufferproc `,
diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst
index 4bf3cfe100cd013..456f7d89bca03cd 100644
--- a/Doc/c-api/bytearray.rst
+++ b/Doc/c-api/bytearray.rst
@@ -5,7 +5,7 @@
Byte Array Objects
------------------
-.. index:: object: bytearray
+.. index:: pair: object; bytearray
.. c:type:: PyByteArrayObject
diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst
index d62962cab45f6bb..4790d3b2da43754 100644
--- a/Doc/c-api/bytes.rst
+++ b/Doc/c-api/bytes.rst
@@ -8,7 +8,7 @@ Bytes Objects
These functions raise :exc:`TypeError` when expecting a bytes parameter and
called with a non-bytes parameter.
-.. index:: object: bytes
+.. index:: pair: object; bytes
.. c:type:: PyBytesObject
@@ -64,39 +64,39 @@ called with a non-bytes parameter.
+-------------------+---------------+--------------------------------+
| Format Characters | Type | Comment |
+===================+===============+================================+
- | :attr:`%%` | *n/a* | The literal % character. |
+ | ``%%`` | *n/a* | The literal % character. |
+-------------------+---------------+--------------------------------+
- | :attr:`%c` | int | A single byte, |
+ | ``%c`` | int | A single byte, |
| | | represented as a C int. |
+-------------------+---------------+--------------------------------+
- | :attr:`%d` | int | Equivalent to |
+ | ``%d`` | int | Equivalent to |
| | | ``printf("%d")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%u` | unsigned int | Equivalent to |
+ | ``%u`` | unsigned int | Equivalent to |
| | | ``printf("%u")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%ld` | long | Equivalent to |
+ | ``%ld`` | long | Equivalent to |
| | | ``printf("%ld")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%lu` | unsigned long | Equivalent to |
+ | ``%lu`` | unsigned long | Equivalent to |
| | | ``printf("%lu")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%zd` | :c:type:`\ | Equivalent to |
+ | ``%zd`` | :c:type:`\ | Equivalent to |
| | Py_ssize_t` | ``printf("%zd")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%zu` | size_t | Equivalent to |
+ | ``%zu`` | size_t | Equivalent to |
| | | ``printf("%zu")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%i` | int | Equivalent to |
+ | ``%i`` | int | Equivalent to |
| | | ``printf("%i")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%x` | int | Equivalent to |
+ | ``%x`` | int | Equivalent to |
| | | ``printf("%x")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%s` | const char\* | A null-terminated C character |
+ | ``%s`` | const char\* | A null-terminated C character |
| | | array. |
+-------------------+---------------+--------------------------------+
- | :attr:`%p` | const void\* | The hex representation of a C |
+ | ``%p`` | const void\* | The hex representation of a C |
| | | pointer. Mostly equivalent to |
| | | ``printf("%p")`` except that |
| | | it is guaranteed to start with |
@@ -155,6 +155,7 @@ called with a non-bytes parameter.
Return the null-terminated contents of the object *obj*
through the output variables *buffer* and *length*.
+ Returns ``0`` on success.
If *length* is ``NULL``, the bytes object
may not contain embedded null bytes;
@@ -184,8 +185,8 @@ called with a non-bytes parameter.
.. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart)
Create a new bytes object in *\*bytes* containing the contents of *newpart*
- appended to *bytes*. This version decrements the reference count of
- *newpart*.
+ appended to *bytes*. This version releases the :term:`strong reference`
+ to *newpart* (i.e. decrements its reference count).
.. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize)
diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst
index 6fb2e15196103d4..7198d6bc056eb43 100644
--- a/Doc/c-api/call.rst
+++ b/Doc/c-api/call.rst
@@ -59,12 +59,12 @@ This bears repeating:
.. versionchanged:: 3.12
- The :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag is now removed from a class
+ The :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag is now removed from a class
when the class's :py:meth:`~object.__call__` method is reassigned.
(This internally sets :c:member:`~PyTypeObject.tp_call` only, and thus
may make it behave differently than the vectorcall function.)
In earlier Python versions, vectorcall should only be used with
- :const:`immutable ` or static types.
+ :c:macro:`immutable ` or static types.
A class should not implement vectorcall if that would be slower
than *tp_call*. For example, if the callee needs to convert
@@ -72,7 +72,7 @@ the arguments to an args tuple and kwargs dict anyway, then there is no point
in implementing vectorcall.
Classes can implement the vectorcall protocol by enabling the
-:const:`Py_TPFLAGS_HAVE_VECTORCALL` flag and setting
+:c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag and setting
:c:member:`~PyTypeObject.tp_vectorcall_offset` to the offset inside the
object structure where a *vectorcallfunc* appears.
This is a pointer to a function with the following signature:
@@ -84,7 +84,7 @@ This is a pointer to a function with the following signature:
values of the keyword arguments.
This can be *NULL* if there are no arguments.
- *nargsf* is the number of positional arguments plus possibly the
- :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` flag.
+ :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET` flag.
To get the actual number of positional arguments from *nargsf*,
use :c:func:`PyVectorcall_NARGS`.
- *kwnames* is a tuple containing the names of the keyword arguments;
@@ -104,28 +104,17 @@ This is a pointer to a function with the following signature:
``args[0]`` may be changed.
Whenever they can do so cheaply (without additional allocation), callers
- are encouraged to use :const:`PY_VECTORCALL_ARGUMENTS_OFFSET`.
+ are encouraged to use :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET`.
Doing so will allow callables such as bound methods to make their onward
calls (which include a prepended *self* argument) very efficiently.
+ .. versionadded:: 3.8
+
To call an object that implements vectorcall, use a :ref:`call API `
function as with any other callable.
:c:func:`PyObject_Vectorcall` will usually be most efficient.
-.. note::
-
- In CPython 3.8, the vectorcall API and related functions were available
- provisionally under names with a leading underscore:
- ``_PyObject_Vectorcall``, ``_Py_TPFLAGS_HAVE_VECTORCALL``,
- ``_PyObject_VectorcallMethod``, ``_PyVectorcall_Function``,
- ``_PyObject_CallOneArg``, ``_PyObject_CallMethodNoArgs``,
- ``_PyObject_CallMethodOneArg``.
- Additionally, ``PyObject_VectorcallDict`` was available as
- ``_PyObject_FastCallDict``.
- The old names are still defined as aliases of the new, non-underscored names.
-
-
Recursion Control
.................
@@ -165,7 +154,7 @@ Vectorcall Support API
This is mostly useful to check whether or not *op* supports vectorcall,
which can be done by checking ``PyVectorcall_Function(op) != NULL``.
- .. versionadded:: 3.8
+ .. versionadded:: 3.9
.. c:function:: PyObject* PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict)
@@ -174,7 +163,7 @@ Vectorcall Support API
This is a specialized function, intended to be put in the
:c:member:`~PyTypeObject.tp_call` slot or be used in an implementation of ``tp_call``.
- It does not check the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag
+ It does not check the :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag
and it does not fall back to ``tp_call``.
.. versionadded:: 3.8
@@ -392,11 +381,11 @@ please see individual documentation for details.
*args[0]*, and the *args* array starting at *args[1]* represents the arguments
of the call. There must be at least one positional argument.
*nargsf* is the number of positional arguments including *args[0]*,
- plus :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` if the value of ``args[0]`` may
+ plus :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET` if the value of ``args[0]`` may
temporarily be changed. Keyword arguments can be passed just like in
:c:func:`PyObject_Vectorcall`.
- If the object has the :const:`Py_TPFLAGS_METHOD_DESCRIPTOR` feature,
+ If the object has the :c:macro:`Py_TPFLAGS_METHOD_DESCRIPTOR` feature,
this will call the unbound method object with the full
*args* vector as arguments.
diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst
index 1c8f432505ef68a..cdb8aa33e9fd325 100644
--- a/Doc/c-api/capsule.rst
+++ b/Doc/c-api/capsule.rst
@@ -5,7 +5,7 @@
Capsules
--------
-.. index:: object: Capsule
+.. index:: pair: object; Capsule
Refer to :ref:`using-capsules` for more information on using these objects.
@@ -64,7 +64,7 @@ Refer to :ref:`using-capsules` for more information on using these objects.
The *name* parameter must compare exactly to the name stored in the capsule.
If the name stored in the capsule is ``NULL``, the *name* passed in must also
- be ``NULL``. Python uses the C function :c:func:`strcmp` to compare capsule
+ be ``NULL``. Python uses the C function :c:func:`!strcmp` to compare capsule
names.
@@ -121,7 +121,7 @@ Refer to :ref:`using-capsules` for more information on using these objects.
compared.)
In other words, if :c:func:`PyCapsule_IsValid` returns a true value, calls to
- any of the accessors (any function starting with :c:func:`PyCapsule_Get`) are
+ any of the accessors (any function starting with ``PyCapsule_Get``) are
guaranteed to succeed.
Return a nonzero value if the object is valid and matches the name passed in.
diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst
index ac4ef5adc5cc20f..f8cd0344fdd1c0c 100644
--- a/Doc/c-api/cell.rst
+++ b/Doc/c-api/cell.rst
@@ -25,7 +25,7 @@ Cell objects are not likely to be useful elsewhere.
The type object corresponding to cell objects.
-.. c:function:: int PyCell_Check(ob)
+.. c:function:: int PyCell_Check(PyObject *ob)
Return true if *ob* is a cell object; *ob* must not be ``NULL``. This
function always succeeds.
diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst
index 9054e7ee3181a52..5082b0cb6ad3f3e 100644
--- a/Doc/c-api/code.rst
+++ b/Doc/c-api/code.rst
@@ -33,27 +33,46 @@ bound into a function.
Return the number of free variables in *co*.
-.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
+.. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
Return a new code object. If you need a dummy code object to create a frame,
- use :c:func:`PyCode_NewEmpty` instead. Calling :c:func:`PyCode_New` directly
- will bind you to a precise Python version since the definition of the bytecode
- changes often. The many arguments of this function are inter-dependent in complex
+ use :c:func:`PyCode_NewEmpty` instead.
+
+ Since the definition of the bytecode changes often, calling
+ :c:func:`PyUnstable_Code_New` directly can bind you to a precise Python version.
+
+ The many arguments of this function are inter-dependent in complex
ways, meaning that subtle changes to values are likely to result in incorrect
execution or VM crashes. Use this function only with extreme care.
.. versionchanged:: 3.11
- Added ``exceptiontable`` parameter.
+ Added ``qualname`` and ``exceptiontable`` parameters.
+
+ .. index:: single: PyCode_New
+
+ .. versionchanged:: 3.12
-.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
+ Renamed from ``PyCode_New`` as part of :ref:`unstable-c-api`.
+ The old name is deprecated, but will remain available until the
+ signature changes again.
- Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments.
- The same caveats that apply to ``PyCode_New`` also apply to this function.
+.. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
- .. versionadded:: 3.8
+ Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments.
+ The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function.
+
+ .. index:: single: PyCode_NewWithPosOnlyArgs
+
+ .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs``
.. versionchanged:: 3.11
- Added ``exceptiontable`` parameter.
+ Added ``qualname`` and ``exceptiontable`` parameters.
+
+ .. versionchanged:: 3.12
+
+ Renamed to ``PyUnstable_Code_NewWithPosOnlyArgs``.
+ The old name is deprecated, but will remain available until the
+ signature changes again.
.. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
@@ -77,6 +96,8 @@ bound into a function.
Returns ``1`` if the function succeeds and 0 otherwise.
+ .. versionadded:: 3.11
+
.. c:function:: PyObject* PyCode_GetCode(PyCodeObject *co)
Equivalent to the Python code ``getattr(co, 'co_code')``.
@@ -115,3 +136,129 @@ bound into a function.
the free variables. On error, ``NULL`` is returned and an exception is raised.
.. versionadded:: 3.11
+
+.. c:function:: int PyCode_AddWatcher(PyCode_WatchCallback callback)
+
+ Register *callback* as a code object watcher for the current interpreter.
+ Return an ID which may be passed to :c:func:`PyCode_ClearWatcher`.
+ In case of error (e.g. no more watcher IDs available),
+ return ``-1`` and set an exception.
+
+ .. versionadded:: 3.12
+
+.. c:function:: int PyCode_ClearWatcher(int watcher_id)
+
+ Clear watcher identified by *watcher_id* previously returned from
+ :c:func:`PyCode_AddWatcher` for the current interpreter.
+ Return ``0`` on success, or ``-1`` and set an exception on error
+ (e.g. if the given *watcher_id* was never registered.)
+
+ .. versionadded:: 3.12
+
+.. c:type:: PyCodeEvent
+
+ Enumeration of possible code object watcher events:
+ - ``PY_CODE_EVENT_CREATE``
+ - ``PY_CODE_EVENT_DESTROY``
+
+ .. versionadded:: 3.12
+
+.. c:type:: int (*PyCode_WatchCallback)(PyCodeEvent event, PyCodeObject* co)
+
+ Type of a code object watcher callback function.
+
+ If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked
+ after `co` has been fully initialized. Otherwise, the callback is invoked
+ before the destruction of *co* takes place, so the prior state of *co*
+ can be inspected.
+
+ If *event* is ``PY_CODE_EVENT_DESTROY``, taking a reference in the callback
+ to the about-to-be-destroyed code object will resurrect it and prevent it
+ from being freed at this time. When the resurrected object is destroyed
+ later, any watcher callbacks active at that time will be called again.
+
+ Users of this API should not rely on internal runtime implementation
+ details. Such details may include, but are not limited to, the exact
+ order and timing of creation and destruction of code objects. While
+ changes in these details may result in differences observable by watchers
+ (including whether a callback is invoked or not), it does not change
+ the semantics of the Python code being executed.
+
+ If the callback sets an exception, it must return ``-1``; this exception will
+ be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`.
+ Otherwise it should return ``0``.
+
+ There may already be a pending exception set on entry to the callback. In
+ this case, the callback should return ``0`` with the same exception still
+ set. This means the callback may not call any other API that can set an
+ exception unless it saves and clears the exception state first, and restores
+ it before returning.
+
+ .. versionadded:: 3.12
+
+
+Extra information
+-----------------
+
+To support low-level extensions to frame evaluation, such as external
+just-in-time compilers, it is possible to attach arbitrary extra data to
+code objects.
+
+These functions are part of the unstable C API tier:
+this functionality is a CPython implementation detail, and the API
+may change without deprecation warnings.
+
+.. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free)
+
+ Return a new an opaque index value used to adding data to code objects.
+
+ You generally call this function once (per interpreter) and use the result
+ with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate
+ data on individual code objects.
+
+ If *free* is not ``NULL``: when a code object is deallocated,
+ *free* will be called on non-``NULL`` data stored under the new index.
+ Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`.
+
+ .. index:: single: _PyEval_RequestCodeExtraIndex
+
+ .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex``
+
+ .. versionchanged:: 3.12
+
+ Renamed to ``PyUnstable_Eval_RequestCodeExtraIndex``.
+ The old private name is deprecated, but will be available until the API
+ changes.
+
+.. c:function:: int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
+
+ Set *extra* to the extra data stored under the given index.
+ Return 0 on success. Set an exception and return -1 on failure.
+
+ If no data was set under the index, set *extra* to ``NULL`` and return
+ 0 without setting an exception.
+
+ .. index:: single: _PyCode_GetExtra
+
+ .. versionadded:: 3.6 as ``_PyCode_GetExtra``
+
+ .. versionchanged:: 3.12
+
+ Renamed to ``PyUnstable_Code_GetExtra``.
+ The old private name is deprecated, but will be available until the API
+ changes.
+
+.. c:function:: int PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
+
+ Set the extra data stored under the given index to *extra*.
+ Return 0 on success. Set an exception and return -1 on failure.
+
+ .. index:: single: _PyCode_SetExtra
+
+ .. versionadded:: 3.6 as ``_PyCode_SetExtra``
+
+ .. versionchanged:: 3.12
+
+ Renamed to ``PyUnstable_Code_SetExtra``.
+ The old private name is deprecated, but will be available until the API
+ changes.
diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst
index 235c77c945cc5be..8ae5c4fecd6248f 100644
--- a/Doc/c-api/codec.rst
+++ b/Doc/c-api/codec.rst
@@ -7,7 +7,7 @@ Codec registry and support functions
Register a new codec search function.
- As side effect, this tries to load the :mod:`encodings` package, if not yet
+ As side effect, this tries to load the :mod:`!encodings` package, if not yet
done, to make sure that it is always first in the list of search functions.
.. c:function:: int PyCodec_Unregister(PyObject *search_function)
diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst
index 9228ce852000230..e3fd001c599c800 100644
--- a/Doc/c-api/complex.rst
+++ b/Doc/c-api/complex.rst
@@ -5,7 +5,7 @@
Complex Number Objects
----------------------
-.. index:: object: complex number
+.. index:: pair: object; complex number
Python's complex number objects are implemented as two distinct types when
viewed from the C API: one is the Python object exposed to Python programs, and
@@ -64,7 +64,7 @@ pointers. This is consistent throughout the API.
representation.
If *divisor* is null, this method returns zero and sets
- :c:data:`errno` to :c:data:`EDOM`.
+ :c:data:`errno` to :c:macro:`!EDOM`.
.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp)
@@ -73,7 +73,7 @@ pointers. This is consistent throughout the API.
representation.
If *num* is null and *exp* is not a positive real number,
- this method returns zero and sets :c:data:`errno` to :c:data:`EDOM`.
+ this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`.
Complex Numbers as Python Objects
@@ -127,12 +127,12 @@ Complex Numbers as Python Objects
Return the :c:type:`Py_complex` value of the complex number *op*.
- If *op* is not a Python complex number object but has a :meth:`__complex__`
+ If *op* is not a Python complex number object but has a :meth:`~object.__complex__`
method, this method will first be called to convert *op* to a Python complex
- number object. If ``__complex__()`` is not defined then it falls back to
- :meth:`__float__`. If ``__float__()`` is not defined then it falls back
- to :meth:`__index__`. Upon failure, this method returns ``-1.0`` as a real
+ number object. If :meth:`!__complex__` is not defined then it falls back to
+ :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back
+ to :meth:`~object.__index__`. Upon failure, this method returns ``-1.0`` as a real
value.
.. versionchanged:: 3.8
- Use :meth:`__index__` if available.
+ Use :meth:`~object.__index__` if available.
diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst
index 8d3124a12fa9d25..880f7b15ce68e82 100644
--- a/Doc/c-api/concrete.rst
+++ b/Doc/c-api/concrete.rst
@@ -40,7 +40,7 @@ This section describes Python type objects and the singleton object ``None``.
Numeric Objects
===============
-.. index:: object: numeric
+.. index:: pair: object; numeric
.. toctree::
@@ -55,7 +55,7 @@ Numeric Objects
Sequence Objects
================
-.. index:: object: sequence
+.. index:: pair: object; sequence
Generic operations on sequence objects were discussed in the previous chapter;
this section deals with the specific kinds of sequence objects that are
@@ -77,7 +77,7 @@ intrinsic to the Python language.
Container Objects
=================
-.. index:: object: mapping
+.. index:: pair: object; mapping
.. toctree::
diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst
index fdb321fe7ab3f24..c5350123dfdfdcd 100644
--- a/Doc/c-api/conversion.rst
+++ b/Doc/c-api/conversion.rst
@@ -119,10 +119,10 @@ The following functions provide locale-independent string to number conversions.
.. c:function:: int PyOS_stricmp(const char *s1, const char *s2)
Case insensitive comparison of strings. The function works almost
- identically to :c:func:`strcmp` except that it ignores the case.
+ identically to :c:func:`!strcmp` except that it ignores the case.
.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size)
Case insensitive comparison of strings. The function works almost
- identically to :c:func:`strncmp` except that it ignores the case.
+ identically to :c:func:`!strncmp` except that it ignores the case.
diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst
index 72fc07afbf1f4db..97522da773477e8 100644
--- a/Doc/c-api/datetime.rst
+++ b/Doc/c-api/datetime.rst
@@ -8,11 +8,54 @@ DateTime Objects
Various date and time objects are supplied by the :mod:`datetime` module.
Before using any of these functions, the header file :file:`datetime.h` must be
included in your source (note that this is not included by :file:`Python.h`),
-and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of
+and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of
the module initialisation function. The macro puts a pointer to a C structure
-into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following
+into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following
macros.
+.. c:type:: PyDateTime_Date
+
+ This subtype of :c:type:`PyObject` represents a Python date object.
+
+.. c:type:: PyDateTime_DateTime
+
+ This subtype of :c:type:`PyObject` represents a Python datetime object.
+
+.. c:type:: PyDateTime_Time
+
+ This subtype of :c:type:`PyObject` represents a Python time object.
+
+.. c:type:: PyDateTime_Delta
+
+ This subtype of :c:type:`PyObject` represents the difference between two datetime values.
+
+.. c:var:: PyTypeObject PyDateTime_DateType
+
+ This instance of :c:type:`PyTypeObject` represents the Python date type;
+ it is the same object as :class:`datetime.date` in the Python layer.
+
+.. c:var:: PyTypeObject PyDateTime_DateTimeType
+
+ This instance of :c:type:`PyTypeObject` represents the Python datetime type;
+ it is the same object as :class:`datetime.datetime` in the Python layer.
+
+.. c:var:: PyTypeObject PyDateTime_TimeType
+
+ This instance of :c:type:`PyTypeObject` represents the Python time type;
+ it is the same object as :class:`datetime.time` in the Python layer.
+
+.. c:var:: PyTypeObject PyDateTime_DeltaType
+
+ This instance of :c:type:`PyTypeObject` represents Python type for
+ the difference between two datetime values;
+ it is the same object as :class:`datetime.timedelta` in the Python layer.
+
+.. c:var:: PyTypeObject PyDateTime_TZInfoType
+
+ This instance of :c:type:`PyTypeObject` represents the Python time zone info type;
+ it is the same object as :class:`datetime.tzinfo` in the Python layer.
+
+
Macro for access to the UTC singleton:
.. c:var:: PyObject* PyDateTime_TimeZone_UTC
@@ -28,7 +71,7 @@ Type-check macros:
.. c:function:: int PyDate_Check(PyObject *ob)
Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of
- :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. This function always
+ :c:data:`!PyDateTime_DateType`. *ob* must not be ``NULL``. This function always
succeeds.
@@ -41,7 +84,7 @@ Type-check macros:
.. c:function:: int PyDateTime_Check(PyObject *ob)
Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of
- :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always
+ :c:data:`!PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always
succeeds.
@@ -54,7 +97,7 @@ Type-check macros:
.. c:function:: int PyTime_Check(PyObject *ob)
Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of
- :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always
+ :c:data:`!PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always
succeeds.
@@ -67,7 +110,7 @@ Type-check macros:
.. c:function:: int PyDelta_Check(PyObject *ob)
Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of
- :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always
+ :c:data:`!PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always
succeeds.
@@ -80,7 +123,7 @@ Type-check macros:
.. c:function:: int PyTZInfo_Check(PyObject *ob)
Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of
- :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always
+ :c:data:`!PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always
succeeds.
@@ -133,7 +176,7 @@ Macros to create objects:
:class:`datetime.timedelta` objects.
-.. c:function:: PyObject* PyTimeZone_FromOffset(PyDateTime_DeltaType* offset)
+.. c:function:: PyObject* PyTimeZone_FromOffset(PyObject *offset)
Return a :class:`datetime.timezone` object with an unnamed fixed offset
represented by the *offset* argument.
@@ -141,7 +184,7 @@ Macros to create objects:
.. versionadded:: 3.7
-.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyDateTime_DeltaType* offset, PyUnicode* name)
+.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyObject *offset, PyObject *name)
Return a :class:`datetime.timezone` object with a fixed offset represented
by the *offset* argument and with tzname *name*.
@@ -150,8 +193,8 @@ Macros to create objects:
Macros to extract fields from date objects. The argument must be an instance of
-:c:data:`PyDateTime_Date`, including subclasses (such as
-:c:data:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is
+:c:type:`PyDateTime_Date`, including subclasses (such as
+:c:type:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is
not checked:
.. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o)
@@ -170,7 +213,7 @@ not checked:
Macros to extract fields from datetime objects. The argument must be an
-instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument
+instance of :c:type:`PyDateTime_DateTime`, including subclasses. The argument
must not be ``NULL``, and the type is not checked:
.. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o)
@@ -208,7 +251,7 @@ must not be ``NULL``, and the type is not checked:
Macros to extract fields from time objects. The argument must be an instance of
-:c:data:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``,
+:c:type:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``,
and the type is not checked:
.. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o)
@@ -246,7 +289,7 @@ and the type is not checked:
Macros to extract fields from time delta objects. The argument must be an
-instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must
+instance of :c:type:`PyDateTime_Delta`, including subclasses. The argument must
not be ``NULL``, and the type is not checked:
.. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o)
diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst
index e5f28b59a701e00..8471c98d0448721 100644
--- a/Doc/c-api/dict.rst
+++ b/Doc/c-api/dict.rst
@@ -5,7 +5,7 @@
Dictionary Objects
------------------
-.. index:: object: dictionary
+.. index:: pair: object; dictionary
.. c:type:: PyDictObject
@@ -55,6 +55,15 @@ Dictionary Objects
This is equivalent to the Python expression ``key in p``.
+.. c:function:: int PyDict_ContainsString(PyObject *p, const char *key)
+
+ This is the same as :c:func:`PyDict_Contains`, but *key* is specified as a
+ :c:expr:`const char*` UTF-8 encoded bytes string, rather than a
+ :c:expr:`PyObject*`.
+
+ .. versionadded:: 3.13
+
+
.. c:function:: PyObject* PyDict_Copy(PyObject *p)
Return a new dictionary that contains the same key-value pairs as *p*.
@@ -70,17 +79,14 @@ Dictionary Objects
.. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val)
- .. index:: single: PyUnicode_FromString()
-
- Insert *val* into the dictionary *p* using *key* as a key. *key* should
- be a :c:expr:`const char*`. The key object is created using
- ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on
- failure. This function *does not* steal a reference to *val*.
+ This is the same as :c:func:`PyDict_SetItem`, but *key* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
.. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key)
- Remove the entry in dictionary *p* with key *key*. *key* must be hashable;
+ Remove the entry in dictionary *p* with key *key*. *key* must be :term:`hashable`;
if it isn't, :exc:`TypeError` is raised.
If *key* is not in the dictionary, :exc:`KeyError` is raised.
Return ``0`` on success or ``-1`` on failure.
@@ -88,19 +94,37 @@ Dictionary Objects
.. c:function:: int PyDict_DelItemString(PyObject *p, const char *key)
- Remove the entry in dictionary *p* which has a key specified by the string *key*.
- If *key* is not in the dictionary, :exc:`KeyError` is raised.
- Return ``0`` on success or ``-1`` on failure.
+ This is the same as :c:func:`PyDict_DelItem`, but *key* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
+
+
+.. c:function:: int PyDict_GetItemRef(PyObject *p, PyObject *key, PyObject **result)
+
+ Return a new :term:`strong reference` to the object from dictionary *p*
+ which has a key *key*:
+
+ * If the key is present, set *\*result* to a new :term:`strong reference`
+ to the value and return ``1``.
+ * If the key is missing, set *\*result* to ``NULL`` and return ``0``.
+ * On error, raise an exception and return ``-1``.
+
+ .. versionadded:: 3.13
+
+ See also the :c:func:`PyObject_GetItem` function.
.. c:function:: PyObject* PyDict_GetItem(PyObject *p, PyObject *key)
- Return the object from dictionary *p* which has a key *key*. Return ``NULL``
- if the key *key* is not present, but *without* setting an exception.
+ Return a :term:`borrowed reference` to the object from dictionary *p* which
+ has a key *key*. Return ``NULL`` if the key *key* is missing *without*
+ setting an exception.
+
+ .. note::
- Note that exceptions which occur while calling :meth:`__hash__` and
- :meth:`__eq__` methods will get suppressed.
- To get error reporting use :c:func:`PyDict_GetItemWithError()` instead.
+ Exceptions that occur while this calls :meth:`~object.__hash__` and
+ :meth:`~object.__eq__` methods are silently ignored.
+ Prefer the :c:func:`PyDict_GetItemWithError` function instead.
.. versionchanged:: 3.10
Calling this API without :term:`GIL` held had been allowed for historical
@@ -118,12 +142,25 @@ Dictionary Objects
.. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key)
This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a
- :c:expr:`const char*`, rather than a :c:expr:`PyObject*`.
+ :c:expr:`const char*` UTF-8 encoded bytes string, rather than a
+ :c:expr:`PyObject*`.
+
+ .. note::
+
+ Exceptions that occur while this calls :meth:`~object.__hash__` and
+ :meth:`~object.__eq__` methods or while creating the temporary :class:`str`
+ object are silently ignored.
+ Prefer using the :c:func:`PyDict_GetItemWithError` function with your own
+ :c:func:`PyUnicode_FromString` *key* instead.
- Note that exceptions which occur while calling :meth:`__hash__` and
- :meth:`__eq__` methods and creating a temporary string object
- will get suppressed.
- To get error reporting use :c:func:`PyDict_GetItemWithError()` instead.
+
+.. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result)
+
+ Similar than :c:func:`PyDict_GetItemRef`, but *key* is specified as a
+ :c:expr:`const char*` UTF-8 encoded bytes string, rather than a
+ :c:expr:`PyObject*`.
+
+ .. versionadded:: 3.13
.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *defaultobj)
@@ -136,6 +173,33 @@ Dictionary Objects
.. versionadded:: 3.4
+
+.. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result)
+
+ Remove *key* from dictionary *p* and optionally return the removed value.
+ Do not raise :exc:`KeyError` if the key missing.
+
+ - If the key is present, set *\*result* to a new reference to the removed
+ value if *result* is not ``NULL``, and return ``1``.
+ - If the key is missing, set *\*result* to ``NULL`` if *result* is not
+ ``NULL``, and return ``0``.
+ - On error, raise an exception and return ``-1``.
+
+ This is similar to :meth:`dict.pop`, but without the default value and
+ not raising :exc:`KeyError` if the key missing.
+
+ .. versionadded:: 3.13
+
+
+.. c:function:: int PyDict_PopString(PyObject *p, const char *key, PyObject **result)
+
+ Similar to :c:func:`PyDict_Pop`, but *key* is specified as a
+ :c:expr:`const char*` UTF-8 encoded bytes string, rather than a
+ :c:expr:`PyObject*`.
+
+ .. versionadded:: 3.13
+
+
.. c:function:: PyObject* PyDict_Items(PyObject *p)
Return a :c:type:`PyListObject` containing all the items from the dictionary.
@@ -154,7 +218,7 @@ Dictionary Objects
.. c:function:: Py_ssize_t PyDict_Size(PyObject *p)
- .. index:: builtin: len
+ .. index:: pair: built-in function; len
Return the number of items in the dictionary. This is equivalent to
``len(p)`` on a dictionary.
@@ -298,13 +362,26 @@ Dictionary Objects
dictionary.
The callback may inspect but must not modify *dict*; doing so could have
- unpredictable effects, including infinite recursion.
+ unpredictable effects, including infinite recursion. Do not trigger Python
+ code execution in the callback, as it could modify the dict as a side effect.
+
+ If *event* is ``PyDict_EVENT_DEALLOCATED``, taking a new reference in the
+ callback to the about-to-be-destroyed dictionary will resurrect it and
+ prevent it from being freed at this time. When the resurrected object is
+ destroyed later, any watcher callbacks active at that time will be called
+ again.
Callbacks occur before the notified modification to *dict* takes place, so
the prior state of *dict* can be inspected.
- If the callback returns with an exception set, it must return ``-1``; this
- exception will be printed as an unraisable exception using
- :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.
+ If the callback sets an exception, it must return ``-1``; this exception will
+ be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`.
+ Otherwise it should return ``0``.
+
+ There may already be a pending exception set on entry to the callback. In
+ this case, the callback should return ``0`` with the same exception still
+ set. This means the callback may not call any other API that can set an
+ exception unless it saves and clears the exception state first, and restores
+ it before returning.
.. versionadded:: 3.12
diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index 087e0a61d12d59a..a3a63b38c432f2b 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -60,9 +60,14 @@ Printing and clearing
Call this function **only** when the error indicator is set. Otherwise it
will cause a fatal error!
- If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`,
- :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the
- type, value and traceback of the printed exception, respectively.
+ If *set_sys_last_vars* is nonzero, the variable :data:`sys.last_exc` is
+ set to the printed exception. For backwards compatibility, the
+ deprecated variables :data:`sys.last_type`, :data:`sys.last_value` and
+ :data:`sys.last_traceback` are also set to the type, value and traceback
+ of this exception, respectively.
+
+ .. versionchanged:: 3.12
+ The setting of :data:`sys.last_exc` was added.
.. c:function:: void PyErr_Print()
@@ -78,14 +83,41 @@ Printing and clearing
This utility function prints a warning message to ``sys.stderr`` when an
exception has been set but it is impossible for the interpreter to actually
raise the exception. It is used, for example, when an exception occurs in an
- :meth:`__del__` method.
+ :meth:`~object.__del__` method.
The function is called with a single argument *obj* that identifies the context
in which the unraisable exception occurred. If possible,
the repr of *obj* will be printed in the warning message.
+ If *obj* is ``NULL``, only the traceback is printed.
An exception must be set when calling this function.
+ .. versionchanged:: 3.4
+ Print a traceback. Print only traceback if *obj* is ``NULL``.
+
+ .. versionchanged:: 3.8
+ Use :func:`sys.unraisablehook`.
+
+
+.. c:function:: void PyErr_FormatUnraisable(const char *format, ...)
+
+ Similar to :c:func:`PyErr_WriteUnraisable`, but the *format* and subsequent
+ parameters help format the warning message; they have the same meaning and
+ values as in :c:func:`PyUnicode_FromFormat`.
+ ``PyErr_WriteUnraisable(obj)`` is roughtly equivalent to
+ ``PyErr_FormatUnraisable("Exception ignored in: %R, obj)``.
+ If *format* is ``NULL``, only the traceback is printed.
+
+ .. versionadded:: 3.13
+
+
+.. c:function:: void PyErr_DisplayException(PyObject *exc)
+
+ Print the standard traceback display of ``exc`` to ``sys.stderr``, including
+ chained exceptions and notes.
+
+ .. versionadded:: 3.12
+
Raising exceptions
==================
@@ -99,7 +131,8 @@ For convenience, some of these functions will always return a
This is the most common way to set the error indicator. The first argument
specifies the exception type; it is normally one of the standard exceptions,
- e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count.
+ e.g. :c:data:`PyExc_RuntimeError`. You need not create a new
+ :term:`strong reference` to it (e.g. with :c:func:`Py_INCREF`).
The second argument is an error message; it is decoded from ``'utf-8'``.
@@ -152,9 +185,9 @@ For convenience, some of these functions will always return a
This is a convenience function to raise an exception when a C library function
has returned an error and set the C variable :c:data:`errno`. It constructs a
tuple object whose first item is the integer :c:data:`errno` value and whose
- second item is the corresponding error message (gotten from :c:func:`strerror`),
+ second item is the corresponding error message (gotten from :c:func:`!strerror`),
and then calls ``PyErr_SetObject(type, object)``. On Unix, when the
- :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call,
+ :c:data:`errno` value is :c:macro:`!EINTR`, indicating an interrupted system call,
this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator,
leaves it set to that. The function always returns ``NULL``, so a wrapper
function around a system call can write ``return PyErr_SetFromErrno(type);``
@@ -166,7 +199,7 @@ For convenience, some of these functions will always return a
Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if
*filenameObject* is not ``NULL``, it is passed to the constructor of *type* as
a third parameter. In the case of :exc:`OSError` exception,
- this is used to define the :attr:`filename` attribute of the
+ this is used to define the :attr:`!filename` attribute of the
exception instance.
@@ -189,12 +222,12 @@ For convenience, some of these functions will always return a
.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr)
This is a convenience function to raise :exc:`WindowsError`. If called with
- *ierr* of ``0``, the error code returned by a call to :c:func:`GetLastError`
- is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve
- the Windows description of error code given by *ierr* or :c:func:`GetLastError`,
+ *ierr* of ``0``, the error code returned by a call to :c:func:`!GetLastError`
+ is used instead. It calls the Win32 function :c:func:`!FormatMessage` to retrieve
+ the Windows description of error code given by *ierr* or :c:func:`!GetLastError`,
then it constructs a tuple object whose first item is the *ierr* value and whose
second item is the corresponding error message (gotten from
- :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError,
+ :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError,
object)``. This function always returns ``NULL``.
.. availability:: Windows.
@@ -210,17 +243,21 @@ For convenience, some of these functions will always return a
.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename)
- Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the
- filename is given as a C string. *filename* is decoded from the filesystem
- encoding (:func:`os.fsdecode`).
+ Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior
+ that if *filename* is not ``NULL``, it is decoded from the filesystem
+ encoding (:func:`os.fsdecode`) and passed to the constructor of
+ :exc:`OSError` as a third parameter to be used to define the
+ :attr:`!filename` attribute of the exception instance.
.. availability:: Windows.
.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename)
- Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an
- additional parameter specifying the exception type to be raised.
+ Similar to :c:func:`PyErr_SetExcFromWindowsErr`, with the additional behavior
+ that if *filename* is not ``NULL``, it is passed to the constructor of
+ :exc:`OSError` as a third parameter to be used to define the
+ :attr:`!filename` attribute of the exception instance.
.. availability:: Windows.
@@ -400,8 +437,48 @@ Querying the error indicator
recursively in subtuples) are searched for a match.
+.. c:function:: PyObject *PyErr_GetRaisedException(void)
+
+ Return the exception currently being raised, clearing the error indicator at
+ the same time.
+
+ This function is used by code that needs to catch exceptions,
+ or code that needs to save and restore the error indicator temporarily.
+
+ For example::
+
+ {
+ PyObject *exc = PyErr_GetRaisedException();
+
+ /* ... code that might produce other errors ... */
+
+ PyErr_SetRaisedException(exc);
+ }
+
+ .. seealso:: :c:func:`PyErr_GetHandledException`,
+ to save the exception currently being handled.
+
+ .. versionadded:: 3.12
+
+
+.. c:function:: void PyErr_SetRaisedException(PyObject *exc)
+
+ Set *exc* as the exception currently being raised,
+ clearing the existing exception if one is set.
+
+ .. warning::
+
+ This call steals a reference to *exc*, which must be a valid exception.
+
+ .. versionadded:: 3.12
+
+
.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback)
+ .. deprecated:: 3.12
+
+ Use :c:func:`PyErr_GetRaisedException` instead.
+
Retrieve the error indicator into three variables whose addresses are passed.
If the error indicator is not set, set all three variables to ``NULL``. If it is
set, it will be cleared and you own a reference to each object retrieved. The
@@ -409,8 +486,10 @@ Querying the error indicator
.. note::
- This function is normally only used by code that needs to catch exceptions or
- by code that needs to save and restore the error indicator temporarily, e.g.::
+ This function is normally only used by legacy code that needs to catch
+ exceptions or save and restore the error indicator temporarily.
+
+ For example::
{
PyObject *type, *value, *traceback;
@@ -424,8 +503,14 @@ Querying the error indicator
.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback)
- Set the error indicator from the three objects. If the error indicator is
- already set, it is cleared first. If the objects are ``NULL``, the error
+ .. deprecated:: 3.12
+
+ Use :c:func:`PyErr_SetRaisedException` instead.
+
+ Set the error indicator from the three objects,
+ *type*, *value*, and *traceback*,
+ clearing the existing exception if one is set.
+ If the objects are ``NULL``, the error
indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or
traceback. The exception type should be a class. Do not pass an invalid
exception type or value. (Violating these rules will cause subtle problems
@@ -436,13 +521,18 @@ Querying the error indicator
.. note::
- This function is normally only used by code that needs to save and restore the
- error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current
- error indicator.
+ This function is normally only used by legacy code that needs to
+ save and restore the error indicator temporarily.
+ Use :c:func:`PyErr_Fetch` to save the current error indicator.
.. c:function:: void PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb)
+ .. deprecated:: 3.12
+
+ Use :c:func:`PyErr_GetRaisedException` instead,
+ to avoid any possible de-normalization.
+
Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below
can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is
not an instance of the same class. This function can be used to instantiate
@@ -543,7 +633,7 @@ Signal Handling
.. c:function:: int PyErr_CheckSignals()
.. index::
- module: signal
+ pair: module; signal
single: SIGINT
single: KeyboardInterrupt (built-in exception)
@@ -567,18 +657,18 @@ Signal Handling
be interruptible by user requests (such as by pressing Ctrl-C).
.. note::
- The default Python signal handler for :const:`SIGINT` raises the
+ The default Python signal handler for :c:macro:`!SIGINT` raises the
:exc:`KeyboardInterrupt` exception.
.. c:function:: void PyErr_SetInterrupt()
.. index::
- module: signal
+ pair: module; signal
single: SIGINT
single: KeyboardInterrupt (built-in exception)
- Simulate the effect of a :const:`SIGINT` signal arriving.
+ Simulate the effect of a :c:macro:`!SIGINT` signal arriving.
This is equivalent to ``PyErr_SetInterruptEx(SIGINT)``.
.. note::
@@ -589,7 +679,7 @@ Signal Handling
.. c:function:: int PyErr_SetInterruptEx(int signum)
.. index::
- module: signal
+ pair: module; signal
single: KeyboardInterrupt (built-in exception)
Simulate the effect of a signal arriving. The next time
@@ -602,7 +692,7 @@ Signal Handling
to interrupt an operation).
If the given signal isn't handled by Python (it was set to
- :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), it will be ignored.
+ :py:const:`signal.SIG_DFL` or :py:const:`signal.SIG_IGN`), it will be ignored.
If *signum* is outside of the allowed range of signal numbers, ``-1``
is returned. Otherwise, ``0`` is returned. The error indicator is
@@ -690,7 +780,7 @@ Exception Objects
.. c:function:: PyObject* PyException_GetCause(PyObject *ex)
- Return the cause (either an exception instance, or :const:`None`,
+ Return the cause (either an exception instance, or ``None``,
set by ``raise ... from ...``) associated with the exception as a new
reference, as accessible from Python through :attr:`__cause__`.
@@ -699,11 +789,33 @@ Exception Objects
Set the cause associated with the exception to *cause*. Use ``NULL`` to clear
it. There is no type check to make sure that *cause* is either an exception
- instance or :const:`None`. This steals a reference to *cause*.
+ instance or ``None``. This steals a reference to *cause*.
:attr:`__suppress_context__` is implicitly set to ``True`` by this function.
+.. c:function:: PyObject* PyException_GetArgs(PyObject *ex)
+
+ Return :attr:`~BaseException.args` of exception *ex*.
+
+
+.. c:function:: void PyException_SetArgs(PyObject *ex, PyObject *args)
+
+ Set :attr:`~BaseException.args` of exception *ex* to *args*.
+
+.. c:function:: PyObject* PyUnstable_Exc_PrepReraiseStar(PyObject *orig, PyObject *excs)
+
+ Implement part of the interpreter's implementation of :keyword:`!except*`.
+ *orig* is the original exception that was caught, and *excs* is the list of
+ the exceptions that need to be raised. This list contains the unhandled
+ part of *orig*, if any, as well as the exceptions that were raised from the
+ :keyword:`!except*` clauses (so they have a different traceback from *orig*) and
+ those that were reraised (and have the same traceback as *orig*).
+ Return the :exc:`ExceptionGroup` that needs to be reraised in the end, or
+ ``None`` if there is nothing to reraise.
+
+ .. versionadded:: 3.12
+
.. _unicodeexceptions:
Unicode Exception Objects
@@ -788,7 +900,7 @@ because the :ref:`call protocol ` takes care of recursion handling.
Marks a point where a recursive C-level call is about to be performed.
- If :const:`USE_STACKCHECK` is defined, this function checks if the OS
+ If :c:macro:`USE_STACKCHECK` is defined, this function checks if the OS
stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it
sets a :exc:`MemoryError` and returns a nonzero value.
@@ -801,7 +913,7 @@ because the :ref:`call protocol ` takes care of recursion handling.
depth limit.
.. versionchanged:: 3.9
- This function is now also available in the limited API.
+ This function is now also available in the :ref:`limited API `.
.. c:function:: void Py_LeaveRecursiveCall(void)
@@ -809,7 +921,7 @@ because the :ref:`call protocol ` takes care of recursion handling.
*successful* invocation of :c:func:`Py_EnterRecursiveCall`.
.. versionchanged:: 3.9
- This function is now also available in the limited API.
+ This function is now also available in the :ref:`limited API `.
Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires
special recursion handling. In addition to protecting the stack,
diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst
index 58ed58e54668599..b36c800e00444ac 100644
--- a/Doc/c-api/file.rst
+++ b/Doc/c-api/file.rst
@@ -5,7 +5,7 @@
File Objects
------------
-.. index:: object: file
+.. index:: pair: object; file
These APIs are a minimal emulation of the Python 2 C API for built-in file
objects, which used to rely on the buffered I/O (:c:expr:`FILE*`) support
@@ -93,7 +93,7 @@ the :mod:`io` APIs instead.
.. index:: single: Py_PRINT_RAW
Write object *obj* to file object *p*. The only supported flag for *flags* is
- :const:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written
+ :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written
instead of the :func:`repr`. Return ``0`` on success or ``-1`` on failure; the
appropriate exception will be set.
diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst
index 023b12c20b7c83f..4f6ac0d8175c6b2 100644
--- a/Doc/c-api/float.rst
+++ b/Doc/c-api/float.rst
@@ -3,9 +3,9 @@
.. _floatobjects:
Floating Point Objects
-----------------------
+======================
-.. index:: object: floating point
+.. index:: pair: object; floating point
.. c:type:: PyFloatObject
@@ -45,14 +45,14 @@ Floating Point Objects
.. c:function:: double PyFloat_AsDouble(PyObject *pyfloat)
Return a C :c:expr:`double` representation of the contents of *pyfloat*. If
- *pyfloat* is not a Python floating point object but has a :meth:`__float__`
+ *pyfloat* is not a Python floating point object but has a :meth:`~object.__float__`
method, this method will first be called to convert *pyfloat* into a float.
- If ``__float__()`` is not defined then it falls back to :meth:`__index__`.
+ If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`.
This method returns ``-1.0`` upon failure, so one should call
:c:func:`PyErr_Occurred` to check for errors.
.. versionchanged:: 3.8
- Use :meth:`__index__` if available.
+ Use :meth:`~object.__index__` if available.
.. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat)
@@ -79,7 +79,7 @@ Floating Point Objects
Pack and Unpack functions
-=========================
+-------------------------
The pack and unpack functions provide an efficient platform-independent way to
store floating-point values as byte strings. The Pack routines produce a bytes
@@ -104,12 +104,12 @@ happens in such cases is partly accidental (alas).
.. versionadded:: 3.11
Pack functions
---------------
+^^^^^^^^^^^^^^
The pack routines write 2, 4 or 8 bytes, starting at *p*. *le* is an
:c:expr:`int` argument, non-zero if you want the bytes string in little-endian
format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` ``p+7``), zero if you
-want big-endian format (exponent first, at *p*). The :c:data:`PY_BIG_ENDIAN`
+want big-endian format (exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN`
constant can be used to use the native endian: it is equal to ``1`` on big
endian processor, or ``0`` on little endian processor.
@@ -135,12 +135,12 @@ There are two problems on non-IEEE platforms:
Unpack functions
-----------------
+^^^^^^^^^^^^^^^^
The unpack routines read 2, 4 or 8 bytes, starting at *p*. *le* is an
:c:expr:`int` argument, non-zero if the bytes string is in little-endian format
(exponent last, at ``p+1``, ``p+3`` or ``p+6`` and ``p+7``), zero if big-endian
-(exponent first, at *p*). The :c:data:`PY_BIG_ENDIAN` constant can be used to
+(exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` constant can be used to
use the native endian: it is equal to ``1`` on big endian processor, or ``0``
on little endian processor.
diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst
index 46ce700abf14748..1accee2767a4852 100644
--- a/Doc/c-api/frame.rst
+++ b/Doc/c-api/frame.rst
@@ -19,6 +19,24 @@ can be used to get a frame object.
See also :ref:`Reflection `.
+.. c:var:: PyTypeObject PyFrame_Type
+
+ The type of frame objects.
+ It is the same object as :py:class:`types.FrameType` in the Python layer.
+
+ .. versionchanged:: 3.11
+
+ Previously, this type was only available after including
+ ````.
+
+.. c:function:: int PyFrame_Check(PyObject *obj)
+
+ Return non-zero if *obj* is a frame object.
+
+ .. versionchanged:: 3.11
+
+ Previously, this function was only available after including
+ ````.
.. c:function:: PyFrameObject* PyFrame_GetBack(PyFrameObject *frame)
@@ -79,6 +97,27 @@ See also :ref:`Reflection `.
.. versionadded:: 3.11
+.. c:function:: PyObject* PyFrame_GetVar(PyFrameObject *frame, PyObject *name)
+
+ Get the variable *name* of *frame*.
+
+ * Return a :term:`strong reference` to the variable value on success.
+ * Raise :exc:`NameError` and return ``NULL`` if the variable does not exist.
+ * Raise an exception and return ``NULL`` on error.
+
+ *name* type must be a :class:`str`.
+
+ .. versionadded:: 3.12
+
+
+.. c:function:: PyObject* PyFrame_GetVarString(PyFrameObject *frame, const char *name)
+
+ Similar to :c:func:`PyFrame_GetVar`, but the variable name is a C string
+ encoded in UTF-8.
+
+ .. versionadded:: 3.12
+
+
.. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame)
Get the *frame*'s ``f_locals`` attribute (:class:`dict`).
@@ -91,3 +130,38 @@ See also :ref:`Reflection `.
.. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame)
Return the line number that *frame* is currently executing.
+
+
+
+Internal Frames
+^^^^^^^^^^^^^^^
+
+Unless using :pep:`523`, you will not need this.
+
+.. c:struct:: _PyInterpreterFrame
+
+ The interpreter's internal frame representation.
+
+ .. versionadded:: 3.11
+
+.. c:function:: PyObject* PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame);
+
+ Return a :term:`strong reference` to the code object for the frame.
+
+ .. versionadded:: 3.12
+
+
+.. c:function:: int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame);
+
+ Return the byte offset into the last executed instruction.
+
+ .. versionadded:: 3.12
+
+
+.. c:function:: int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame);
+
+ Return the currently executing line number, or -1 if there is no line number.
+
+ .. versionadded:: 3.12
+
+
diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst
index df88e85e518829e..5857dba82c11c64 100644
--- a/Doc/c-api/function.rst
+++ b/Doc/c-api/function.rst
@@ -5,7 +5,7 @@
Function Objects
----------------
-.. index:: object: function
+.. index:: pair: object; function
There are a few functions specific to Python functions.
@@ -118,3 +118,74 @@ There are a few functions specific to Python functions.
must be a dictionary or ``Py_None``.
Raises :exc:`SystemError` and returns ``-1`` on failure.
+
+
+.. c:function:: int PyFunction_AddWatcher(PyFunction_WatchCallback callback)
+
+ Register *callback* as a function watcher for the current interpreter.
+ Return an ID which may be passed to :c:func:`PyFunction_ClearWatcher`.
+ In case of error (e.g. no more watcher IDs available),
+ return ``-1`` and set an exception.
+
+ .. versionadded:: 3.12
+
+
+.. c:function:: int PyFunction_ClearWatcher(int watcher_id)
+
+ Clear watcher identified by *watcher_id* previously returned from
+ :c:func:`PyFunction_AddWatcher` for the current interpreter.
+ Return ``0`` on success, or ``-1`` and set an exception on error
+ (e.g. if the given *watcher_id* was never registered.)
+
+ .. versionadded:: 3.12
+
+
+.. c:type:: PyFunction_WatchEvent
+
+ Enumeration of possible function watcher events:
+ - ``PyFunction_EVENT_CREATE``
+ - ``PyFunction_EVENT_DESTROY``
+ - ``PyFunction_EVENT_MODIFY_CODE``
+ - ``PyFunction_EVENT_MODIFY_DEFAULTS``
+ - ``PyFunction_EVENT_MODIFY_KWDEFAULTS``
+
+ .. versionadded:: 3.12
+
+
+.. c:type:: int (*PyFunction_WatchCallback)(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value)
+
+ Type of a function watcher callback function.
+
+ If *event* is ``PyFunction_EVENT_CREATE`` or ``PyFunction_EVENT_DESTROY``
+ then *new_value* will be ``NULL``. Otherwise, *new_value* will hold a
+ :term:`borrowed reference` to the new value that is about to be stored in
+ *func* for the attribute that is being modified.
+
+ The callback may inspect but must not modify *func*; doing so could have
+ unpredictable effects, including infinite recursion.
+
+ If *event* is ``PyFunction_EVENT_CREATE``, then the callback is invoked
+ after `func` has been fully initialized. Otherwise, the callback is invoked
+ before the modification to *func* takes place, so the prior state of *func*
+ can be inspected. The runtime is permitted to optimize away the creation of
+ function objects when possible. In such cases no event will be emitted.
+ Although this creates the possibility of an observable difference of
+ runtime behavior depending on optimization decisions, it does not change
+ the semantics of the Python code being executed.
+
+ If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the
+ callback to the about-to-be-destroyed function will resurrect it, preventing
+ it from being freed at this time. When the resurrected object is destroyed
+ later, any watcher callbacks active at that time will be called again.
+
+ If the callback sets an exception, it must return ``-1``; this exception will
+ be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`.
+ Otherwise it should return ``0``.
+
+ There may already be a pending exception set on entry to the callback. In
+ this case, the callback should return ``0`` with the same exception still
+ set. This means the callback may not call any other API that can set an
+ exception unless it saves and clears the exception state first, and restores
+ it before returning.
+
+ .. versionadded:: 3.12
diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst
index 8c90d1e8991c100..6b2494ee4f0ed4c 100644
--- a/Doc/c-api/gcsupport.rst
+++ b/Doc/c-api/gcsupport.rst
@@ -13,22 +13,20 @@ or strings), do not need to provide any explicit support for garbage
collection.
To create a container type, the :c:member:`~PyTypeObject.tp_flags` field of the type object must
-include the :const:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the
+include the :c:macro:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the
:c:member:`~PyTypeObject.tp_traverse` handler. If instances of the type are mutable, a
:c:member:`~PyTypeObject.tp_clear` implementation must also be provided.
-.. data:: Py_TPFLAGS_HAVE_GC
- :noindex:
-
+:c:macro:`Py_TPFLAGS_HAVE_GC`
Objects with a type with this flag set must conform with the rules
documented here. For convenience these objects will be referred to as
container objects.
Constructors for container types must conform to two rules:
-#. The memory for the object must be allocated using :c:func:`PyObject_GC_New`
- or :c:func:`PyObject_GC_NewVar`.
+#. The memory for the object must be allocated using :c:macro:`PyObject_GC_New`
+ or :c:macro:`PyObject_GC_NewVar`.
#. Once all the fields which may contain references to other containers are
initialized, it must call :c:func:`PyObject_GC_Track`.
@@ -52,23 +50,42 @@ rules:
:c:member:`~PyTypeObject.tp_flags`, :c:member:`~PyTypeObject.tp_traverse`
and :c:member:`~PyTypeObject.tp_clear` fields if the type inherits from a
class that implements the garbage collector protocol and the child class
- does *not* include the :const:`Py_TPFLAGS_HAVE_GC` flag.
+ does *not* include the :c:macro:`Py_TPFLAGS_HAVE_GC` flag.
+
+.. c:macro:: PyObject_GC_New(TYPE, typeobj)
+
+ Analogous to :c:macro:`PyObject_New` but for container objects with the
+ :c:macro:`Py_TPFLAGS_HAVE_GC` flag set.
+
+.. c:macro:: PyObject_GC_NewVar(TYPE, typeobj, size)
+
+ Analogous to :c:macro:`PyObject_NewVar` but for container objects with the
+ :c:macro:`Py_TPFLAGS_HAVE_GC` flag set.
-.. c:function:: TYPE* PyObject_GC_New(TYPE, PyTypeObject *type)
+.. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size)
- Analogous to :c:func:`PyObject_New` but for container objects with the
- :const:`Py_TPFLAGS_HAVE_GC` flag set.
+ Analogous to :c:macro:`PyObject_GC_New` but allocates *extra_size*
+ bytes at the end of the object (at offset
+ :c:member:`~PyTypeObject.tp_basicsize`).
+ The allocated memory is initialized to zeros,
+ except for the :c:type:`Python object header `.
+ The extra data will be deallocated with the object, but otherwise it is
+ not managed by Python.
-.. c:function:: TYPE* PyObject_GC_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size)
+ .. warning::
+ The function is marked as unstable because the final mechanism
+ for reserving extra data after an instance is not yet decided.
+ For allocating a variable number of fields, prefer using
+ :c:type:`PyVarObject` and :c:member:`~PyTypeObject.tp_itemsize`
+ instead.
- Analogous to :c:func:`PyObject_NewVar` but for container objects with the
- :const:`Py_TPFLAGS_HAVE_GC` flag set.
+ .. versionadded:: 3.12
.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize)
- Resize an object allocated by :c:func:`PyObject_NewVar`. Returns the
+ Resize an object allocated by :c:macro:`PyObject_NewVar`. Returns the
resized object or ``NULL`` on failure. *op* must not be tracked by the collector yet.
@@ -111,8 +128,8 @@ rules:
.. c:function:: void PyObject_GC_Del(void *op)
- Releases memory allocated to an object using :c:func:`PyObject_GC_New` or
- :c:func:`PyObject_GC_NewVar`.
+ Releases memory allocated to an object using :c:macro:`PyObject_GC_New` or
+ :c:macro:`PyObject_GC_NewVar`.
.. c:function:: void PyObject_GC_UnTrack(void *op)
@@ -126,7 +143,7 @@ rules:
.. versionchanged:: 3.8
- The :c:func:`_PyObject_GC_TRACK` and :c:func:`_PyObject_GC_UNTRACK` macros
+ The :c:func:`!_PyObject_GC_TRACK` and :c:func:`!_PyObject_GC_UNTRACK` macros
have been removed from the public C API.
The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter of this type:
@@ -228,3 +245,36 @@ garbage collection runs.
Returns the current state, 0 for disabled and 1 for enabled.
.. versionadded:: 3.10
+
+
+Querying Garbage Collector State
+--------------------------------
+
+The C-API provides the following interface for querying information about
+the garbage collector.
+
+.. c:function:: void PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg)
+
+ Run supplied *callback* on all live GC-capable objects. *arg* is passed through to
+ all invocations of *callback*.
+
+ .. warning::
+ If new objects are (de)allocated by the callback it is undefined if they
+ will be visited.
+
+ Garbage collection is disabled during operation. Explicitly running a collection
+ in the callback may lead to undefined behaviour e.g. visiting the same objects
+ multiple times or not at all.
+
+ .. versionadded:: 3.12
+
+.. c:type:: int (*gcvisitobjects_t)(PyObject *object, void *arg)
+
+ Type of the visitor function to be passed to :c:func:`PyUnstable_GC_VisitObjects`.
+ *arg* is the same as the *arg* passed to ``PyUnstable_GC_VisitObjects``.
+ Return ``0`` to continue iteration, return ``1`` to stop iteration. Other return
+ values are reserved for now so behavior on returning anything else is undefined.
+
+ .. versionadded:: 3.12
+
+
diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst
new file mode 100644
index 000000000000000..4dc121d7fbaa9b4
--- /dev/null
+++ b/Doc/c-api/hash.rst
@@ -0,0 +1,48 @@
+.. highlight:: c
+
+PyHash API
+----------
+
+See also the :c:member:`PyTypeObject.tp_hash` member.
+
+.. c:type:: Py_hash_t
+
+ Hash value type: signed integer.
+
+ .. versionadded:: 3.2
+
+.. c:type:: Py_uhash_t
+
+ Hash value type: unsigned integer.
+
+ .. versionadded:: 3.2
+
+
+.. c:type:: PyHash_FuncDef
+
+ Hash function definition used by :c:func:`PyHash_GetFuncDef`.
+
+ .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t)
+
+ Hash function.
+
+ .. c:member:: const char *name
+
+ Hash function name (UTF-8 encoded string).
+
+ .. c:member:: const int hash_bits
+
+ Internal size of the hash value in bits.
+
+ .. c:member:: const int seed_bits
+
+ Size of seed input in bits.
+
+ .. versionadded:: 3.4
+
+
+.. c:function:: PyHash_FuncDef* PyHash_GetFuncDef(void)
+
+ Get the hash function definition.
+
+ .. versionadded:: 3.4
diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst
index a51619db6d3d97c..137780cc359cf9d 100644
--- a/Doc/c-api/import.rst
+++ b/Doc/c-api/import.rst
@@ -38,10 +38,13 @@ Importing Modules
to per-module locks for most purposes, so this function's special
behaviour isn't needed anymore.
+ .. deprecated-removed:: 3.13 3.15
+ Use :c:func:`PyImport_ImportModule` instead.
+
.. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist)
- .. index:: builtin: __import__
+ .. index:: pair: built-in function; __import__
Import a module. This is best described by referring to the built-in Python
function :func:`__import__`.
@@ -95,50 +98,63 @@ Importing Modules
an exception set on failure (the module still exists in this case).
-.. c:function:: PyObject* PyImport_AddModuleObject(PyObject *name)
+.. c:function:: PyObject* PyImport_AddModuleRef(const char *name)
+
+ Return the module object corresponding to a module name.
+
+ The *name* argument may be of the form ``package.module``. First check the
+ modules dictionary if there's one there, and if not, create a new one and
+ insert it in the modules dictionary.
+
+ Return a :term:`strong reference` to the module on success. Return ``NULL``
+ with an exception set on failure.
+
+ The module name *name* is decoded from UTF-8.
+
+ This function does not load or import the module; if the module wasn't
+ already loaded, you will get an empty module object. Use
+ :c:func:`PyImport_ImportModule` or one of its variants to import a module.
+ Package structures implied by a dotted name for *name* are not created if
+ not already present.
- Return the module object corresponding to a module name. The *name* argument
- may be of the form ``package.module``. First check the modules dictionary if
- there's one there, and if not, create a new one and insert it in the modules
- dictionary. Return ``NULL`` with an exception set on failure.
+ .. versionadded:: 3.13
- .. note::
- This function does not load or import the module; if the module wasn't already
- loaded, you will get an empty module object. Use :c:func:`PyImport_ImportModule`
- or one of its variants to import a module. Package structures implied by a
- dotted name for *name* are not created if not already present.
+.. c:function:: PyObject* PyImport_AddModuleObject(PyObject *name)
+
+ Similar to :c:func:`PyImport_AddModuleRef`, but return a :term:`borrowed
+ reference` and *name* is a Python :class:`str` object.
.. versionadded:: 3.3
.. c:function:: PyObject* PyImport_AddModule(const char *name)
- Similar to :c:func:`PyImport_AddModuleObject`, but the name is a UTF-8
- encoded string instead of a Unicode object.
+ Similar to :c:func:`PyImport_AddModuleRef`, but return a :term:`borrowed
+ reference`.
.. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co)
- .. index:: builtin: compile
+ .. index:: pair: built-in function; compile
Given a module name (possibly of the form ``package.module``) and a code object
read from a Python bytecode file or obtained from the built-in function
:func:`compile`, load the module. Return a new reference to the module object,
or ``NULL`` with an exception set if an error occurred. *name*
- is removed from :attr:`sys.modules` in error cases, even if *name* was already
- in :attr:`sys.modules` on entry to :c:func:`PyImport_ExecCodeModule`. Leaving
- incompletely initialized modules in :attr:`sys.modules` is dangerous, as imports of
+ is removed from :data:`sys.modules` in error cases, even if *name* was already
+ in :data:`sys.modules` on entry to :c:func:`PyImport_ExecCodeModule`. Leaving
+ incompletely initialized modules in :data:`sys.modules` is dangerous, as imports of
such modules have no way to know that the module object is an unknown (and
probably damaged with respect to the module author's intents) state.
The module's :attr:`__spec__` and :attr:`__loader__` will be set, if
not set already, with the appropriate values. The spec's loader will
be set to the module's ``__loader__`` (if set) and to an instance of
- :class:`SourceFileLoader` otherwise.
+ :class:`~importlib.machinery.SourceFileLoader` otherwise.
The module's :attr:`__file__` attribute will be set to the code object's
- :c:member:`co_filename`. If applicable, :attr:`__cached__` will also
+ :attr:`!co_filename`. If applicable, :attr:`__cached__` will also
be set.
This function will reload the module if it was already imported. See
@@ -186,8 +202,10 @@ Importing Modules
.. versionadded:: 3.2
.. versionchanged:: 3.3
- Uses :func:`imp.source_from_cache()` in calculating the source path if
+ Uses :func:`!imp.source_from_cache()` in calculating the source path if
only the bytecode path is provided.
+ .. versionchanged:: 3.12
+ No longer uses the removed :mod:`!imp` module.
.. c:function:: long PyImport_GetMagicNumber()
@@ -223,7 +241,7 @@ Importing Modules
.. c:function:: PyObject* PyImport_GetImporter(PyObject *path)
- Return a finder object for a :data:`sys.path`/:attr:`pkg.__path__` item
+ Return a finder object for a :data:`sys.path`/:attr:`!pkg.__path__` item
*path*, possibly by fetching it from the :data:`sys.path_importer_cache`
dict. If it wasn't yet cached, traverse :data:`sys.path_hooks` until a hook
is found that can handle the path item. Return ``None`` if no hook could;
@@ -292,23 +310,25 @@ Importing Modules
.. c:struct:: _inittab
- Structure describing a single entry in the list of built-in modules. Each of
- these structures gives the name and initialization function for a module built
- into the interpreter. The name is an ASCII encoded string. Programs which
+ Structure describing a single entry in the list of built-in modules.
+ Programs which
embed Python may use an array of these structures in conjunction with
:c:func:`PyImport_ExtendInittab` to provide additional built-in modules.
- The structure is defined in :file:`Include/import.h` as::
+ The structure consists of two members:
- struct _inittab {
- const char *name; /* ASCII encoded string */
- PyObject* (*initfunc)(void);
- };
+ .. c:member:: const char *name
+
+ The module name, as an ASCII encoded string.
+
+ .. c: member:: PyObject* (*initfunc)(void)
+
+ Initialization function for a module built into the interpreter.
.. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab)
Add a collection of modules to the table of built-in modules. The *newtab*
- array must end with a sentinel entry which contains ``NULL`` for the :attr:`name`
+ array must end with a sentinel entry which contains ``NULL`` for the :c:member:`~_inittab.name`
field; failure to provide the sentinel value can result in a memory fault.
Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to
extend the internal table. In the event of failure, no modules are added to the
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index afb17719a77ab25..e89641f74c7491e 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -25,16 +25,10 @@ The following functions can be safely called before Python is initialized:
* :c:func:`PyImport_AppendInittab`
* :c:func:`PyImport_ExtendInittab`
- * :c:func:`PyInitFrozenExtensions`
+ * :c:func:`!PyInitFrozenExtensions`
* :c:func:`PyMem_SetAllocator`
* :c:func:`PyMem_SetupDebugHooks`
* :c:func:`PyObject_SetArenaAllocator`
- * :c:func:`Py_SetPath`
- * :c:func:`Py_SetProgramName`
- * :c:func:`Py_SetPythonHome`
- * :c:func:`Py_SetStandardStreamEncoding`
- * :c:func:`PySys_AddWarnOption`
- * :c:func:`PySys_AddXOption`
* :c:func:`PySys_ResetWarnOptions`
* Informative functions:
@@ -65,7 +59,7 @@ The following functions can be safely called before Python is initialized:
:c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`,
:c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`,
:c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`,
- :c:func:`Py_GetProgramName` and :c:func:`PyEval_InitThreads`.
+ and :c:func:`Py_GetProgramName`.
.. _global-conf-vars:
@@ -93,7 +87,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Set by the :option:`-b` option.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_DebugFlag
@@ -107,7 +101,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment
variable.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_DontWriteBytecodeFlag
@@ -121,7 +115,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE`
environment variable.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_FrozenFlag
@@ -134,7 +128,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Private flag used by ``_freeze_module`` and ``frozenmain`` programs.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_HashRandomizationFlag
@@ -149,7 +143,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment
variable to initialize the secret hash seed.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_IgnoreEnvironmentFlag
@@ -157,12 +151,12 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
:c:member:`PyConfig.use_environment` should be used instead, see
:ref:`Python Initialization Configuration `.
- Ignore all :envvar:`PYTHON*` environment variables, e.g.
+ Ignore all :envvar:`!PYTHON*` environment variables, e.g.
:envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set.
Set by the :option:`-E` and :option:`-I` options.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_InspectFlag
@@ -177,7 +171,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment
variable.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_InteractiveFlag
@@ -202,7 +196,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
.. versionadded:: 3.4
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_LegacyWindowsFSEncodingFlag
@@ -221,7 +215,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
.. availability:: Windows.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_LegacyWindowsStdioFlag
@@ -230,7 +224,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
:ref:`Python Initialization Configuration `.
If the flag is non-zero, use :class:`io.FileIO` instead of
- :class:`WindowsConsoleIO` for :mod:`sys` standard streams.
+ :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams.
Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment
variable is set to a non-empty string.
@@ -239,7 +233,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
.. availability:: Windows.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_NoSiteFlag
@@ -254,7 +248,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Set by the :option:`-S` option.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_NoUserSiteDirectory
@@ -268,7 +262,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Set by the :option:`-s` and :option:`-I` options, and the
:envvar:`PYTHONNOUSERSITE` environment variable.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_OptimizeFlag
@@ -279,7 +273,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment
variable.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_QuietFlag
@@ -293,7 +287,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
.. versionadded:: 3.2
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_UnbufferedStdioFlag
@@ -306,7 +300,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED`
environment variable.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
.. c:var:: int Py_VerboseFlag
@@ -322,7 +316,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment
variable.
- .. deprecated:: 3.12
+ .. deprecated-removed:: 3.12 3.14
Initializing and finalizing the interpreter
@@ -332,16 +326,12 @@ Initializing and finalizing the interpreter
.. c:function:: void Py_Initialize()
.. index::
- single: Py_SetProgramName()
- single: PyEval_InitThreads()
single: modules (in module sys)
single: path (in module sys)
- module: builtins
- module: __main__
- module: sys
+ pair: module; builtins
+ pair: module; __main__
+ pair: module; sys
triple: module; search; path
- single: PySys_SetArgv()
- single: PySys_SetArgvEx()
single: Py_FinalizeEx()
Initialize the Python interpreter. In an application embedding Python,
@@ -352,7 +342,9 @@ Initializing and finalizing the interpreter
the table of loaded modules (``sys.modules``), and creates the fundamental
modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes
the module search path (``sys.path``). It does not set ``sys.argv``; use
- :c:func:`PySys_SetArgvEx` for that. This is a no-op when called for a second time
+ the new :c:type:`PyConfig` API of the :ref:`Python Initialization
+ Configuration ` for that. This is a no-op when called for a
+ second time
(without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a
fatal error if the initialization fails.
@@ -381,6 +373,14 @@ Initializing and finalizing the interpreter
:c:func:`Py_Initialize` is called again.
+.. c:function:: int Py_IsFinalizing()
+
+ Return true (non-zero) if the main Python interpreter is
+ :term:`shutting down `. Return false (zero) otherwise.
+
+ .. versionadded:: 3.13
+
+
.. c:function:: int Py_FinalizeEx()
Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of
@@ -401,7 +401,7 @@ Initializing and finalizing the interpreter
the application.
**Bugs and caveats:** The destruction of modules and objects in modules is done
- in random order; this may cause destructors (:meth:`__del__` methods) to fail
+ in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail
when they depend on other objects (even functions) or modules. Dynamically
loaded extension modules loaded by Python are not unloaded. Small amounts of
memory allocated by the Python interpreter may not be freed (if you find a leak,
@@ -425,76 +425,9 @@ Process-wide parameters
=======================
-.. c:function:: int Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
-
- .. index::
- single: Py_Initialize()
- single: main()
- triple: stdin; stdout; sdterr
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`
- should be used instead, see :ref:`Python Initialization Configuration
- `.
-
- This function should be called before :c:func:`Py_Initialize`, if it is
- called at all. It specifies which encoding and error handling to use
- with standard IO, with the same meanings as in :func:`str.encode`.
-
- It overrides :envvar:`PYTHONIOENCODING` values, and allows embedding code
- to control IO encoding when the environment variable does not work.
-
- *encoding* and/or *errors* may be ``NULL`` to use
- :envvar:`PYTHONIOENCODING` and/or default values (depending on other
- settings).
-
- Note that :data:`sys.stderr` always uses the "backslashreplace" error
- handler, regardless of this (or any other) setting.
-
- If :c:func:`Py_FinalizeEx` is called, this function will need to be called
- again in order to affect subsequent calls to :c:func:`Py_Initialize`.
-
- Returns ``0`` if successful, a nonzero value on error (e.g. calling after the
- interpreter has already been initialized).
-
- .. versionadded:: 3.4
-
- .. deprecated:: 3.11
-
-
-.. c:function:: void Py_SetProgramName(const wchar_t *name)
-
- .. index::
- single: Py_Initialize()
- single: main()
- single: Py_GetPath()
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python
- Initialization Configuration `.
-
- This function should be called before :c:func:`Py_Initialize` is called for
- the first time, if it is called at all. It tells the interpreter the value
- of the ``argv[0]`` argument to the :c:func:`main` function of the program
- (converted to wide characters).
- This is used by :c:func:`Py_GetPath` and some other functions below to find
- the Python run-time libraries relative to the interpreter executable. The
- default value is ``'python'``. The argument should point to a
- zero-terminated wide character string in static storage whose contents will not
- change for the duration of the program's execution. No code in the Python
- interpreter will change the contents of this storage.
-
- Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
- :c:expr:`wchar_*` string.
-
- .. deprecated:: 3.11
-
-
-.. c:function:: wchar* Py_GetProgramName()
-
- .. index:: single: Py_SetProgramName()
+.. c:function:: wchar_t* Py_GetProgramName()
- Return the program name set with :c:func:`Py_SetProgramName`, or the default.
+ Return the program name set with :c:member:`PyConfig.program_name`, or the default.
The returned string points into static storage; the caller should not modify its
value.
@@ -504,16 +437,19 @@ Process-wide parameters
.. versionchanged:: 3.10
It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+ .. deprecated-removed:: 3.13 3.15
+ Get :data:`sys.executable` instead.
+
.. c:function:: wchar_t* Py_GetPrefix()
Return the *prefix* for installed platform-independent files. This is derived
through a number of complicated rules from the program name set with
- :c:func:`Py_SetProgramName` and some environment variables; for example, if the
+ :c:member:`PyConfig.program_name` and some environment variables; for example, if the
program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The
returned string points into static storage; the caller should not modify its
value. This corresponds to the :makevar:`prefix` variable in the top-level
- :file:`Makefile` and the ``--prefix`` argument to the :program:`configure`
+ :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure`
script at build time. The value is available to Python code as ``sys.prefix``.
It is only useful on Unix. See also the next function.
@@ -523,12 +459,15 @@ Process-wide parameters
.. versionchanged:: 3.10
It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+ .. deprecated-removed:: 3.13 3.15
+ Get :data:`sys.prefix` instead.
+
.. c:function:: wchar_t* Py_GetExecPrefix()
Return the *exec-prefix* for installed platform-*dependent* files. This is
derived through a number of complicated rules from the program name set with
- :c:func:`Py_SetProgramName` and some environment variables; for example, if the
+ :c:member:`PyConfig.program_name` and some environment variables; for example, if the
program name is ``'/usr/local/bin/python'``, the exec-prefix is
``'/usr/local'``. The returned string points into static storage; the caller
should not modify its value. This corresponds to the :makevar:`exec_prefix`
@@ -564,16 +503,18 @@ Process-wide parameters
.. versionchanged:: 3.10
It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+ .. deprecated-removed:: 3.13 3.15
+ Get :data:`sys.exec_prefix` instead.
+
.. c:function:: wchar_t* Py_GetProgramFullPath()
.. index::
- single: Py_SetProgramName()
single: executable (in module sys)
Return the full program name of the Python executable; this is computed as a
side-effect of deriving the default module search path from the program name
- (set by :c:func:`Py_SetProgramName` above). The returned string points into
+ (set by :c:member:`PyConfig.program_name`). The returned string points into
static storage; the caller should not modify its value. The value is available
to Python code as ``sys.executable``.
@@ -583,16 +524,18 @@ Process-wide parameters
.. versionchanged:: 3.10
It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+ .. deprecated-removed:: 3.13 3.15
+ Get :data:`sys.executable` instead.
+
.. c:function:: wchar_t* Py_GetPath()
.. index::
triple: module; search; path
single: path (in module sys)
- single: Py_SetPath()
Return the default module search path; this is computed from the program name
- (set by :c:func:`Py_SetProgramName` above) and some environment variables.
+ (set by :c:member:`PyConfig.program_name`) and some environment variables.
The returned string consists of a series of directory names separated by a
platform dependent delimiter character. The delimiter character is ``':'``
on Unix and macOS, ``';'`` on Windows. The returned string points into
@@ -609,43 +552,8 @@ Process-wide parameters
.. versionchanged:: 3.10
It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
-
-.. c:function:: void Py_SetPath(const wchar_t *)
-
- .. index::
- triple: module; search; path
- single: path (in module sys)
- single: Py_GetPath()
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.module_search_paths` and
- :c:member:`PyConfig.module_search_paths_set` should be used instead, see
- :ref:`Python Initialization Configuration `.
-
- Set the default module search path. If this function is called before
- :c:func:`Py_Initialize`, then :c:func:`Py_GetPath` won't attempt to compute a
- default search path but uses the one provided instead. This is useful if
- Python is embedded by an application that has full knowledge of the location
- of all modules. The path components should be separated by the platform
- dependent delimiter character, which is ``':'`` on Unix and macOS, ``';'``
- on Windows.
-
- This also causes :data:`sys.executable` to be set to the program
- full path (see :c:func:`Py_GetProgramFullPath`) and for :data:`sys.prefix` and
- :data:`sys.exec_prefix` to be empty. It is up to the caller to modify these
- if required after calling :c:func:`Py_Initialize`.
-
- Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
- :c:expr:`wchar_*` string.
-
- The path argument is copied internally, so the caller may free it after the
- call completes.
-
- .. versionchanged:: 3.8
- The program full path is now used for :data:`sys.executable`, instead
- of the program name.
-
- .. deprecated:: 3.11
+ .. deprecated-removed:: 3.13 3.15
+ Get :data:`sys.path` instead.
.. c:function:: const char* Py_GetVersion()
@@ -718,110 +626,10 @@ Process-wide parameters
``sys.version``.
-.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath)
-
- .. index::
- single: main()
- single: Py_FatalError()
- single: argv (in module sys)
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and
- :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python
- Initialization Configuration `.
-
- Set :data:`sys.argv` based on *argc* and *argv*. These parameters are
- similar to those passed to the program's :c:func:`main` function with the
- difference that the first entry should refer to the script file to be
- executed rather than the executable hosting the Python interpreter. If there
- isn't a script that will be run, the first entry in *argv* can be an empty
- string. If this function fails to initialize :data:`sys.argv`, a fatal
- condition is signalled using :c:func:`Py_FatalError`.
+.. c:function:: wchar_t* Py_GetPythonHome()
- If *updatepath* is zero, this is all the function does. If *updatepath*
- is non-zero, the function also modifies :data:`sys.path` according to the
- following algorithm:
-
- - If the name of an existing script is passed in ``argv[0]``, the absolute
- path of the directory where the script is located is prepended to
- :data:`sys.path`.
- - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point
- to an existing file name), an empty string is prepended to
- :data:`sys.path`, which is the same as prepending the current working
- directory (``"."``).
-
- Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
- :c:expr:`wchar_*` string.
-
- See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv`
- members of the :ref:`Python Initialization Configuration `.
-
- .. note::
- It is recommended that applications embedding the Python interpreter
- for purposes other than executing a single script pass ``0`` as *updatepath*,
- and update :data:`sys.path` themselves if desired.
- See `CVE-2008-5983 `_.
-
- On versions before 3.1.3, you can achieve the same effect by manually
- popping the first :data:`sys.path` element after having called
- :c:func:`PySys_SetArgv`, for example using::
-
- PyRun_SimpleString("import sys; sys.path.pop(0)\n");
-
- .. versionadded:: 3.1.3
-
- .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params;
- check w/ Guido.
-
- .. deprecated:: 3.11
-
-
-.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv)
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used
- instead, see :ref:`Python Initialization Configuration `.
-
- This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set
- to ``1`` unless the :program:`python` interpreter was started with the
- :option:`-I`.
-
- Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
- :c:expr:`wchar_*` string.
-
- See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv`
- members of the :ref:`Python Initialization Configuration `.
-
- .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`.
-
- .. deprecated:: 3.11
-
-
-.. c:function:: void Py_SetPythonHome(const wchar_t *home)
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.home` should be used instead, see :ref:`Python
- Initialization Configuration `.
-
- Set the default "home" directory, that is, the location of the standard
- Python libraries. See :envvar:`PYTHONHOME` for the meaning of the
- argument string.
-
- The argument should point to a zero-terminated character string in static
- storage whose contents will not change for the duration of the program's
- execution. No code in the Python interpreter will change the contents of
- this storage.
-
- Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
- :c:expr:`wchar_*` string.
-
- .. deprecated:: 3.11
-
-
-.. c:function:: w_char* Py_GetPythonHome()
-
- Return the default "home", that is, the value set by a previous call to
- :c:func:`Py_SetPythonHome`, or the value of the :envvar:`PYTHONHOME`
+ Return the default "home", that is, the value set by
+ :c:member:`PyConfig.home`, or the value of the :envvar:`PYTHONHOME`
environment variable if it is set.
This function should not be called before :c:func:`Py_Initialize`, otherwise
@@ -830,6 +638,10 @@ Process-wide parameters
.. versionchanged:: 3.10
It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
+ .. deprecated-removed:: 3.13 3.15
+ Get :c:member:`PyConfig.home` or :envvar:`PYTHONHOME` environment
+ variable instead.
+
.. _threads:
@@ -981,7 +793,7 @@ the fork, and releasing them afterwards. In addition, it resets any
:ref:`lock-objects` in the child. When extending or embedding Python, there
is no way to inform Python of additional (non-Python) locks that need to be
acquired before or reset after a fork. OS facilities such as
-:c:func:`pthread_atfork` would need to be used to accomplish the same thing.
+:c:func:`!pthread_atfork` would need to be used to accomplish the same thing.
Additionally, when extending or embedding Python, calling :c:func:`fork`
directly rather than through :func:`os.fork` (and returning to or calling
into Python) may result in a deadlock by one of Python's internal locks
@@ -1023,47 +835,11 @@ code, or when embedding the Python interpreter:
.. c:type:: PyThreadState
This data structure represents the state of a single thread. The only public
- data member is :attr:`interp` (:c:expr:`PyInterpreterState *`), which points to
- this thread's interpreter state.
-
-
-.. c:function:: void PyEval_InitThreads()
-
- .. index::
- single: PyEval_AcquireThread()
- single: PyEval_ReleaseThread()
- single: PyEval_SaveThread()
- single: PyEval_RestoreThread()
-
- Deprecated function which does nothing.
+ data member is:
- In Python 3.6 and older, this function created the GIL if it didn't exist.
-
- .. versionchanged:: 3.9
- The function now does nothing.
-
- .. versionchanged:: 3.7
- This function is now called by :c:func:`Py_Initialize()`, so you don't
- have to call it yourself anymore.
-
- .. versionchanged:: 3.2
- This function cannot be called before :c:func:`Py_Initialize()` anymore.
-
- .. deprecated-removed:: 3.9 3.11
-
- .. index:: module: _thread
-
-
-.. c:function:: int PyEval_ThreadsInitialized()
-
- Returns a non-zero value if :c:func:`PyEval_InitThreads` has been called. This
- function can be called without holding the GIL, and therefore can be used to
- avoid calls to the locking API when running single-threaded.
-
- .. versionchanged:: 3.7
- The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`.
+ .. c:member:: PyInterpreterState *interp
- .. deprecated-removed:: 3.9 3.11
+ This thread's interpreter state.
.. c:function:: PyThreadState* PyEval_SaveThread()
@@ -1084,7 +860,7 @@ code, or when embedding the Python interpreter:
.. note::
Calling this function from a thread when the runtime is finalizing
will terminate the thread, even if the thread was not created by Python.
- You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to
+ You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to
check if the interpreter is in process of being finalized before calling
this function to avoid unwanted termination.
@@ -1094,6 +870,19 @@ code, or when embedding the Python interpreter:
When the current thread state is ``NULL``, this issues a fatal error (so that
the caller needn't check for ``NULL``).
+ See also :c:func:`PyThreadState_GetUnchecked`.
+
+
+.. c:function:: PyThreadState* PyThreadState_GetUnchecked()
+
+ Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a
+ fatal error if it is NULL. The caller is responsible to check if the result
+ is NULL.
+
+ .. versionadded:: 3.13
+ In Python 3.5 to 3.12, the function was private and known as
+ ``_PyThreadState_UncheckedGet()``.
+
.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate)
@@ -1130,7 +919,7 @@ with sub-interpreters:
.. note::
Calling this function from a thread when the runtime is finalizing
will terminate the thread, even if the thread was not created by Python.
- You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to
+ You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to
check if the interpreter is in process of being finalized before calling
this function to avoid unwanted termination.
@@ -1396,7 +1185,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
function does not steal any references to *exc*. To prevent naive misuse, you
must write your own C extension to call this. Must be called with the GIL held.
Returns the number of thread states modified; this is normally one, but will be
- zero if the thread id isn't found. If *exc* is :const:`NULL`, the pending
+ zero if the thread id isn't found. If *exc* is ``NULL``, the pending
exception (if any) for the thread is cleared. This raises no exceptions.
.. versionchanged:: 3.7
@@ -1412,7 +1201,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
.. note::
Calling this function from a thread when the runtime is finalizing
will terminate the thread, even if the thread was not created by Python.
- You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to
+ You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to
check if the interpreter is in process of being finalized before calling
this function to avoid unwanted termination.
@@ -1437,39 +1226,6 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
available (even when threads have not been initialized).
-.. c:function:: void PyEval_AcquireLock()
-
- Acquire the global interpreter lock. The lock must have been created earlier.
- If this thread already has the lock, a deadlock ensues.
-
- .. deprecated:: 3.2
- This function does not update the current thread state. Please use
- :c:func:`PyEval_RestoreThread` or :c:func:`PyEval_AcquireThread`
- instead.
-
- .. note::
- Calling this function from a thread when the runtime is finalizing
- will terminate the thread, even if the thread was not created by Python.
- You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to
- check if the interpreter is in process of being finalized before calling
- this function to avoid unwanted termination.
-
- .. versionchanged:: 3.8
- Updated to be consistent with :c:func:`PyEval_RestoreThread`,
- :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`,
- and terminate the current thread if called while the interpreter is finalizing.
-
-
-.. c:function:: void PyEval_ReleaseLock()
-
- Release the global interpreter lock. The lock must have been created earlier.
-
- .. deprecated:: 3.2
- This function does not update the current thread state. Please use
- :c:func:`PyEval_SaveThread` or :c:func:`PyEval_ReleaseThread`
- instead.
-
-
.. _sub-interpreter-support:
Sub-interpreter support
@@ -1491,12 +1247,101 @@ You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap`
function. You can create and destroy them using the following functions:
-.. c:function:: PyThreadState* Py_NewInterpreter()
+.. c:type:: PyInterpreterConfig
+
+ Structure containing most parameters to configure a sub-interpreter.
+ Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and
+ never modified by the runtime.
+
+ .. versionadded:: 3.12
+
+ Structure fields:
+
+ .. c:member:: int use_main_obmalloc
+
+ If this is ``0`` then the sub-interpreter will use its own
+ "object" allocator state.
+ Otherwise it will use (share) the main interpreter's.
+
+ If this is ``0`` then
+ :c:member:`~PyInterpreterConfig.check_multi_interp_extensions`
+ must be ``1`` (non-zero).
+ If this is ``1`` then :c:member:`~PyInterpreterConfig.gil`
+ must not be :c:macro:`PyInterpreterConfig_OWN_GIL`.
+
+ .. c:member:: int allow_fork
+
+ If this is ``0`` then the runtime will not support forking the
+ process in any thread where the sub-interpreter is currently active.
+ Otherwise fork is unrestricted.
+
+ Note that the :mod:`subprocess` module still works
+ when fork is disallowed.
+
+ .. c:member:: int allow_exec
+
+ If this is ``0`` then the runtime will not support replacing the
+ current process via exec (e.g. :func:`os.execv`) in any thread
+ where the sub-interpreter is currently active.
+ Otherwise exec is unrestricted.
+
+ Note that the :mod:`subprocess` module still works
+ when exec is disallowed.
+
+ .. c:member:: int allow_threads
+
+ If this is ``0`` then the sub-interpreter's :mod:`threading` module
+ won't create threads.
+ Otherwise threads are allowed.
+
+ .. c:member:: int allow_daemon_threads
+
+ If this is ``0`` then the sub-interpreter's :mod:`threading` module
+ won't create daemon threads.
+ Otherwise daemon threads are allowed (as long as
+ :c:member:`~PyInterpreterConfig.allow_threads` is non-zero).
+
+ .. c:member:: int check_multi_interp_extensions
+
+ If this is ``0`` then all extension modules may be imported,
+ including legacy (single-phase init) modules,
+ in any thread where the sub-interpreter is currently active.
+ Otherwise only multi-phase init extension modules
+ (see :pep:`489`) may be imported.
+ (Also see :c:macro:`Py_mod_multiple_interpreters`.)
+
+ This must be ``1`` (non-zero) if
+ :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``.
+
+ .. c:member:: int gil
+
+ This determines the operation of the GIL for the sub-interpreter.
+ It may be one of the following:
+
+ .. c:namespace:: NULL
+
+ .. c:macro:: PyInterpreterConfig_DEFAULT_GIL
+
+ Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`).
+
+ .. c:macro:: PyInterpreterConfig_SHARED_GIL
+
+ Use (share) the main interpreter's GIL.
+
+ .. c:macro:: PyInterpreterConfig_OWN_GIL
+
+ Use the sub-interpreter's own GIL.
+
+ If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then
+ :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``.
+
+
+.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config)
.. index::
- module: builtins
- module: __main__
- module: sys
+ pair: module; builtins
+ pair: module; __main__
+ pair: module; sys
single: stdout (in module sys)
single: stderr (in module sys)
single: stdin (in module sys)
@@ -1511,16 +1356,47 @@ function. You can create and destroy them using the following functions:
``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying
file descriptors).
- The return value points to the first thread state created in the new
+ The given *config* controls the options with which the interpreter
+ is initialized.
+
+ Upon success, *tstate_p* will be set to the first thread state
+ created in the new
sub-interpreter. This thread state is made in the current thread state.
Note that no actual thread is created; see the discussion of thread states
- below. If creation of the new interpreter is unsuccessful, ``NULL`` is
- returned; no exception is set since the exception state is stored in the
- current thread state and there may not be a current thread state. (Like all
- other Python/C API functions, the global interpreter lock must be held before
- calling this function and is still held when it returns; however, unlike most
- other Python/C API functions, there needn't be a current thread state on
- entry.)
+ below. If creation of the new interpreter is unsuccessful,
+ *tstate_p* is set to ``NULL``;
+ no exception is set since the exception state is stored in the
+ current thread state and there may not be a current thread state.
+
+ Like all other Python/C API functions, the global interpreter lock
+ must be held before calling this function and is still held when it
+ returns. Likewise a current thread state must be set on entry. On
+ success, the returned thread state will be set as current. If the
+ sub-interpreter is created with its own GIL then the GIL of the
+ calling interpreter will be released. When the function returns,
+ the new interpreter's GIL will be held by the current thread and
+ the previously interpreter's GIL will remain released here.
+
+ .. versionadded:: 3.12
+
+ Sub-interpreters are most effective when isolated from each other,
+ with certain functionality restricted::
+
+ PyInterpreterConfig config = {
+ .use_main_obmalloc = 0,
+ .allow_fork = 0,
+ .allow_exec = 0,
+ .allow_threads = 1,
+ .allow_daemon_threads = 0,
+ .check_multi_interp_extensions = 1,
+ .gil = PyInterpreterConfig_OWN_GIL,
+ };
+ PyThreadState *tstate = Py_NewInterpreterFromConfig(&config);
+
+ Note that the config is used only briefly and does not get modified.
+ During initialization the config's values are converted into various
+ :c:type:`PyInterpreterState` values. A read-only copy of the config
+ may be stored internally on the :c:type:`PyInterpreterState`.
.. index::
single: Py_FinalizeEx()
@@ -1555,19 +1431,79 @@ function. You can create and destroy them using the following functions:
.. index:: single: close() (in module os)
+.. c:function:: PyThreadState* Py_NewInterpreter(void)
+
+ .. index::
+ pair: module; builtins
+ pair: module; __main__
+ pair: module; sys
+ single: stdout (in module sys)
+ single: stderr (in module sys)
+ single: stdin (in module sys)
+
+ Create a new sub-interpreter. This is essentially just a wrapper
+ around :c:func:`Py_NewInterpreterFromConfig` with a config that
+ preserves the existing behavior. The result is an unisolated
+ sub-interpreter that shares the main interpreter's GIL, allows
+ fork/exec, allows daemon threads, and allows single-phase init
+ modules.
+
+
.. c:function:: void Py_EndInterpreter(PyThreadState *tstate)
.. index:: single: Py_FinalizeEx()
- Destroy the (sub-)interpreter represented by the given thread state. The given
- thread state must be the current thread state. See the discussion of thread
- states below. When the call returns, the current thread state is ``NULL``. All
- thread states associated with this interpreter are destroyed. (The global
- interpreter lock must be held before calling this function and is still held
- when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that
+ Destroy the (sub-)interpreter represented by the given thread state.
+ The given thread state must be the current thread state. See the
+ discussion of thread states below. When the call returns,
+ the current thread state is ``NULL``. All thread states associated
+ with this interpreter are destroyed. The global interpreter lock
+ used by the target interpreter must be held before calling this
+ function. No GIL is held when it returns.
+
+ :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that
haven't been explicitly destroyed at that point.
+A Per-Interpreter GIL
+---------------------
+
+Using :c:func:`Py_NewInterpreterFromConfig` you can create
+a sub-interpreter that is completely isolated from other interpreters,
+including having its own GIL. The most important benefit of this
+isolation is that such an interpreter can execute Python code without
+being blocked by other interpreters or blocking any others. Thus a
+single Python process can truly take advantage of multiple CPU cores
+when running Python code. The isolation also encourages a different
+approach to concurrency than that of just using threads.
+(See :pep:`554`.)
+
+Using an isolated interpreter requires vigilance in preserving that
+isolation. That especially means not sharing any objects or mutable
+state without guarantees about thread-safety. Even objects that are
+otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared
+because of the refcount. One simple but less-efficient approach around
+this is to use a global lock around all use of some state (or object).
+Alternately, effectively immutable objects (like integers or strings)
+can be made safe in spite of their refcounts by making them :term:`immortal`.
+In fact, this has been done for the builtin singletons, small integers,
+and a number of other builtin objects.
+
+If you preserve isolation then you will have access to proper multi-core
+computing without the complications that come with free-threading.
+Failure to preserve isolation will expose you to the full consequences
+of free-threading, including races and hard-to-debug crashes.
+
+Aside from that, one of the main challenges of using multiple isolated
+interpreters is how to communicate between them safely (not break
+isolation) and efficiently. The runtime and stdlib do not provide
+any standard approach to this yet. A future stdlib module would help
+mitigate the effort of preserving isolation and expose effective tools
+for communicating (and sharing) data between interpreters.
+
+.. versionadded:: 3.12
+
+
Bugs and caveats
----------------
@@ -1675,32 +1611,32 @@ Python-level trace functions in previous versions.
The type of the trace function registered using :c:func:`PyEval_SetProfile` and
:c:func:`PyEval_SetTrace`. The first parameter is the object passed to the
registration function as *obj*, *frame* is the frame object to which the event
- pertains, *what* is one of the constants :const:`PyTrace_CALL`,
- :const:`PyTrace_EXCEPTION`, :const:`PyTrace_LINE`, :const:`PyTrace_RETURN`,
- :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, :const:`PyTrace_C_RETURN`,
- or :const:`PyTrace_OPCODE`, and *arg* depends on the value of *what*:
-
- +------------------------------+----------------------------------------+
- | Value of *what* | Meaning of *arg* |
- +==============================+========================================+
- | :const:`PyTrace_CALL` | Always :c:data:`Py_None`. |
- +------------------------------+----------------------------------------+
- | :const:`PyTrace_EXCEPTION` | Exception information as returned by |
- | | :func:`sys.exc_info`. |
- +------------------------------+----------------------------------------+
- | :const:`PyTrace_LINE` | Always :c:data:`Py_None`. |
- +------------------------------+----------------------------------------+
- | :const:`PyTrace_RETURN` | Value being returned to the caller, |
- | | or ``NULL`` if caused by an exception. |
- +------------------------------+----------------------------------------+
- | :const:`PyTrace_C_CALL` | Function object being called. |
- +------------------------------+----------------------------------------+
- | :const:`PyTrace_C_EXCEPTION` | Function object being called. |
- +------------------------------+----------------------------------------+
- | :const:`PyTrace_C_RETURN` | Function object being called. |
- +------------------------------+----------------------------------------+
- | :const:`PyTrace_OPCODE` | Always :c:data:`Py_None`. |
- +------------------------------+----------------------------------------+
+ pertains, *what* is one of the constants :c:data:`PyTrace_CALL`,
+ :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`,
+ :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`,
+ or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*:
+
+ +-------------------------------+----------------------------------------+
+ | Value of *what* | Meaning of *arg* |
+ +===============================+========================================+
+ | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. |
+ +-------------------------------+----------------------------------------+
+ | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by |
+ | | :func:`sys.exc_info`. |
+ +-------------------------------+----------------------------------------+
+ | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. |
+ +-------------------------------+----------------------------------------+
+ | :c:data:`PyTrace_RETURN` | Value being returned to the caller, |
+ | | or ``NULL`` if caused by an exception. |
+ +-------------------------------+----------------------------------------+
+ | :c:data:`PyTrace_C_CALL` | Function object being called. |
+ +-------------------------------+----------------------------------------+
+ | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. |
+ +-------------------------------+----------------------------------------+
+ | :c:data:`PyTrace_C_RETURN` | Function object being called. |
+ +-------------------------------+----------------------------------------+
+ | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. |
+ +-------------------------------+----------------------------------------+
.. c:var:: int PyTrace_CALL
@@ -1767,8 +1703,8 @@ Python-level trace functions in previous versions.
function as its first parameter, and may be any Python object, or ``NULL``. If
the profile function needs to maintain state, using a different value for *obj*
for each thread provides a convenient and thread-safe place to store it. The
- profile function is called for all monitored events except :const:`PyTrace_LINE`
- :const:`PyTrace_OPCODE` and :const:`PyTrace_EXCEPTION`.
+ profile function is called for all monitored events except :c:data:`PyTrace_LINE`
+ :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`.
See also the :func:`sys.setprofile` function.
@@ -1793,8 +1729,8 @@ Python-level trace functions in previous versions.
:c:func:`PyEval_SetProfile`, except the tracing function does receive line-number
events and per-opcode events, but does not receive any event related to C function
objects being called. Any trace function registered using :c:func:`PyEval_SetTrace`
- will not receive :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or
- :const:`PyTrace_C_RETURN` as a value for the *what* parameter.
+ will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or
+ :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter.
See also the :func:`sys.settrace` function.
diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index 64bdfefd6494ff5..47a8fbb2cd9c97e 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -82,6 +82,8 @@ PyWideStringList
If *length* is non-zero, *items* must be non-``NULL`` and all strings must be
non-``NULL``.
+ .. c:namespace:: NULL
+
Methods:
.. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item)
@@ -101,6 +103,8 @@ PyWideStringList
Python must be preinitialized to call this function.
+ .. c:namespace:: PyWideStringList
+
Structure fields:
.. c:member:: Py_ssize_t length
@@ -135,6 +139,8 @@ PyStatus
Name of the function which created an error, can be ``NULL``.
+ .. c:namespace:: NULL
+
Functions to create a status:
.. c:function:: PyStatus PyStatus_Ok(void)
@@ -210,6 +216,8 @@ PyPreConfig
Structure used to preinitialize Python.
+ .. c:namespace:: NULL
+
Function to initialize a preconfiguration:
.. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
@@ -222,6 +230,8 @@ PyPreConfig
Initialize the preconfiguration with :ref:`Isolated Configuration
`.
+ .. c:namespace:: PyPreConfig
+
Structure fields:
.. c:member:: int allocator
@@ -243,18 +253,28 @@ PyPreConfig
* ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc
memory allocator ` with :ref:`debug hooks
`.
+ * ``PYMEM_ALLOCATOR_MIMALLOC`` (``6``): use ``mimalloc``, a fast
+ malloc replacement.
+ * ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` (``7``): use ``mimalloc``, a fast
+ malloc replacement with :ref:`debug hooks `.
+
``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` are
not supported if Python is :option:`configured using --without-pymalloc
<--without-pymalloc>`.
+ ``PYMEM_ALLOCATOR_MIMALLOC`` and ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` are
+ not supported if Python is :option:`configured using --without-mimalloc
+ <--without-mimalloc>` or if the underlying atomic support isn't
+ available.
+
See :ref:`Memory Management `.
Default: ``PYMEM_ALLOCATOR_NOT_SET``.
.. c:member:: int configure_locale
- Set the LC_CTYPE locale to the user preferred locale?
+ Set the LC_CTYPE locale to the user preferred locale.
If equals to ``0``, set :c:member:`~PyPreConfig.coerce_c_locale` and
:c:member:`~PyPreConfig.coerce_c_locale_warn` members to ``0``.
@@ -429,6 +449,8 @@ PyConfig
When done, the :c:func:`PyConfig_Clear` function must be used to release the
configuration memory.
+ .. c:namespace:: NULL
+
Structure methods:
.. c:function:: void PyConfig_InitPythonConfig(PyConfig *config)
@@ -522,16 +544,28 @@ PyConfig
Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv`
is used, this method must be called before other methods, since the
preinitialization configuration depends on command line arguments (if
- :c:member:`parse_argv` is non-zero).
+ :c:member:`~PyConfig.parse_argv` is non-zero).
The caller of these methods is responsible to handle exceptions (error or
exit) using ``PyStatus_Exception()`` and ``Py_ExitStatusException()``.
+ .. c:namespace:: PyConfig
+
Structure fields:
.. c:member:: PyWideStringList argv
- Command line arguments: :data:`sys.argv`.
+ .. index::
+ single: main()
+ single: argv (in module sys)
+
+ Set :data:`sys.argv` command line arguments based on
+ :c:member:`~PyConfig.argv`. These parameters are similar to those passed
+ to the program's :c:func:`main` function with the difference that the
+ first entry should refer to the script file to be executed rather than
+ the executable hosting the Python interpreter. If there isn't a script
+ that will be run, the first entry in :c:member:`~PyConfig.argv` can be an
+ empty string.
Set :c:member:`~PyConfig.parse_argv` to ``1`` to parse
:c:member:`~PyConfig.argv` the same way the regular Python parses Python
@@ -572,6 +606,8 @@ PyConfig
Part of the :ref:`Python Path Configuration ` output.
+ See also :c:member:`PyConfig.exec_prefix`.
+
.. c:member:: wchar_t* base_executable
Python base executable: :data:`sys._base_executable`.
@@ -584,6 +620,8 @@ PyConfig
Part of the :ref:`Python Path Configuration ` output.
+ See also :c:member:`PyConfig.executable`.
+
.. c:member:: wchar_t* base_prefix
:data:`sys.base_prefix`.
@@ -592,6 +630,8 @@ PyConfig
Part of the :ref:`Python Path Configuration ` output.
+ See also :c:member:`PyConfig.prefix`.
+
.. c:member:: int buffered_stdio
If equals to ``0`` and :c:member:`~PyConfig.configure_c_stdio` is non-zero,
@@ -686,7 +726,7 @@ PyConfig
Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable.
- Need a special build of Python with the ``Py_TRACE_REFS`` macro defined:
+ Needs a special build of Python with the ``Py_TRACE_REFS`` macro defined:
see the :option:`configure --with-trace-refs option <--with-trace-refs>`.
Default: ``0``.
@@ -700,6 +740,8 @@ PyConfig
Part of the :ref:`Python Path Configuration ` output.
+ See also :c:member:`PyConfig.base_exec_prefix`.
+
.. c:member:: wchar_t* executable
The absolute path of the executable binary for the Python interpreter:
@@ -709,6 +751,8 @@ PyConfig
Part of the :ref:`Python Path Configuration ` output.
+ See also :c:member:`PyConfig.base_executable`.
+
.. c:member:: int faulthandler
Enable faulthandler?
@@ -780,10 +824,8 @@ PyConfig
.. c:member:: wchar_t* home
- Python home directory.
-
- If :c:func:`Py_SetPythonHome` has been called, use its argument if it is
- not ``NULL``.
+ Set the default Python "home" directory, that is, the location of the
+ standard Python libraries (see :envvar:`PYTHONHOME`).
Set by the :envvar:`PYTHONHOME` environment variable.
@@ -839,13 +881,26 @@ PyConfig
will produce an error.
Configured by the :option:`-X int_max_str_digits <-X>` command line
- flag or the :envvar:`PYTHONINTMAXSTRDIGITS` environment varable.
+ flag or the :envvar:`PYTHONINTMAXSTRDIGITS` environment variable.
Default: ``-1`` in Python mode. 4300
(:data:`sys.int_info.default_max_str_digits`) in isolated mode.
.. versionadded:: 3.12
+ .. c:member:: int cpu_count
+
+ If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will
+ override the return values of :func:`os.cpu_count`,
+ :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`.
+
+ Configured by the :samp:`-X cpu_count={n|default}` command line
+ flag or the :envvar:`PYTHON_CPU_COUNT` environment variable.
+
+ Default: ``-1``.
+
+ .. versionadded:: 3.13
+
.. c:member:: int isolated
If greater than ``0``, enable isolated mode:
@@ -871,7 +926,7 @@ PyConfig
.. c:member:: int legacy_windows_stdio
If non-zero, use :class:`io.FileIO` instead of
- :class:`io.WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout`
+ :class:`!io._WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout`
and :data:`sys.stderr`.
Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment
@@ -920,7 +975,7 @@ PyConfig
.. c:member:: wchar_t* pythonpath_env
Module search paths (:data:`sys.path`) as a string separated by ``DELIM``
- (:data:`os.path.pathsep`).
+ (:data:`os.pathsep`).
Set by the :envvar:`PYTHONPATH` environment variable.
@@ -1003,7 +1058,7 @@ PyConfig
Incremented by the :option:`-d` command line option. Set to the
:envvar:`PYTHONDEBUG` environment variable value.
- Need a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro
+ Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro
must be defined).
Default: ``0``.
@@ -1029,12 +1084,13 @@ PyConfig
Part of the :ref:`Python Path Configuration ` output.
+ See also :c:member:`PyConfig.base_prefix`.
+
.. c:member:: wchar_t* program_name
Program name used to initialize :c:member:`~PyConfig.executable` and in
early error messages during Python initialization.
- * If :func:`Py_SetProgramName` has been called, use its argument.
* On macOS, use :envvar:`PYTHONEXECUTABLE` environment variable if set.
* If the ``WITH_NEXT_FRAMEWORK`` macro is defined, use
:envvar:`__PYVENV_LAUNCHER__` environment variable if set.
@@ -1054,6 +1110,7 @@ PyConfig
Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and
the :envvar:`PYTHONPYCACHEPREFIX` environment variable.
+ The command-line option takes precedence.
If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``.
@@ -1097,13 +1154,27 @@ PyConfig
Default: ``NULL``.
+ .. c:member:: wchar_t* run_presite
+
+ ``package.module`` path to module that should be imported before
+ ``site.py`` is run.
+
+ Set by the :option:`-X presite=package.module <-X>` command-line
+ option and the :envvar:`PYTHON_PRESITE` environment variable.
+ The command-line option takes precedence.
+
+ Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro
+ must be defined).
+
+ Default: ``NULL``.
+
.. c:member:: int show_ref_count
- Show total reference count at exit?
+ Show total reference count at exit (excluding :term:`immortal` objects)?
Set to ``1`` by :option:`-X showrefcount <-X>` command line option.
- Need a :ref:`debug build of Python ` (the ``Py_REF_DEBUG``
+ Needs a :ref:`debug build of Python ` (the ``Py_REF_DEBUG``
macro must be defined).
Default: ``0``.
@@ -1120,7 +1191,7 @@ PyConfig
Set to ``0`` by the :option:`-S` command line option.
- :data:`sys.flags.no_site` is set to the inverted value of
+ :data:`sys.flags.no_site ` is set to the inverted value of
:c:member:`~PyConfig.site_import`.
Default: ``1``.
@@ -1144,9 +1215,6 @@ PyConfig
:data:`sys.stderr` (but :data:`sys.stderr` always uses
``"backslashreplace"`` error handler).
- If :c:func:`Py_SetStandardStreamEncoding` has been called, use its
- *error* and *errors* arguments if they are not ``NULL``.
-
Use the :envvar:`PYTHONIOENCODING` environment variable if it is
non-empty.
@@ -1162,6 +1230,8 @@ PyConfig
or if the LC_CTYPE locale is "C" or "POSIX".
* ``"strict"`` otherwise.
+ See also :c:member:`PyConfig.legacy_windows_stdio`.
+
.. c:member:: int tracemalloc
Enable tracemalloc?
@@ -1509,7 +1579,7 @@ If a ``._pth`` file is present:
* Set :c:member:`~PyConfig.safe_path` to ``1``.
The ``__PYVENV_LAUNCHER__`` environment variable is used to set
-:c:member:`PyConfig.base_executable`
+:c:member:`PyConfig.base_executable`.
Py_RunMain()
@@ -1582,7 +1652,7 @@ applied during the "Main" phase. It may allow to customize Python in Python to
override or tune the :ref:`Path Configuration `, maybe
install a custom :data:`sys.meta_path` importer or an import hook, etc.
-It may become possible to calculatin the :ref:`Path Configuration
+It may become possible to calculate the :ref:`Path Configuration
` in Python, after the Core phase and before the Main phase,
which is one of the :pep:`432` motivation.
diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst
index 85eb24a495b640d..4dbca92b18b5cd2 100644
--- a/Doc/c-api/intro.rst
+++ b/Doc/c-api/intro.rst
@@ -78,19 +78,19 @@ used by extension writers. Structure member names do not have a reserved prefix.
The header files are typically installed with Python. On Unix, these are
located in the directories :file:`{prefix}/include/pythonversion/` and
-:file:`{exec_prefix}/include/pythonversion/`, where :envvar:`prefix` and
-:envvar:`exec_prefix` are defined by the corresponding parameters to Python's
+:file:`{exec_prefix}/include/pythonversion/`, where :option:`prefix <--prefix>` and
+:option:`exec_prefix <--exec-prefix>` are defined by the corresponding parameters to Python's
:program:`configure` script and *version* is
``'%d.%d' % sys.version_info[:2]``. On Windows, the headers are installed
-in :file:`{prefix}/include`, where :envvar:`prefix` is the installation
+in :file:`{prefix}/include`, where ``prefix`` is the installation
directory specified to the installer.
To include the headers, place both directories (if different) on your compiler's
search path for includes. Do *not* place the parent directories on the search
path and then use ``#include ``; this will break on
multi-platform builds since the platform independent headers under
-:envvar:`prefix` include the platform specific headers from
-:envvar:`exec_prefix`.
+:option:`prefix <--prefix>` include the platform specific headers from
+:option:`exec_prefix <--exec-prefix>`.
C++ users should note that although the API is defined entirely using C, the
header files properly declare the entry points to be ``extern "C"``. As a result,
@@ -105,6 +105,30 @@ defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`).
Others of a more general utility are defined here. This is not necessarily a
complete listing.
+.. c:macro:: PyMODINIT_FUNC
+
+ Declare an extension module ``PyInit`` initialization function. The function
+ return type is :c:expr:`PyObject*`. The macro declares any special linkage
+ declarations required by the platform, and for C++ declares the function as
+ ``extern "C"``.
+
+ The initialization function must be named :samp:`PyInit_{name}`, where
+ *name* is the name of the module, and should be the only non-\ ``static``
+ item defined in the module file. Example::
+
+ static struct PyModuleDef spam_module = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "spam",
+ ...
+ };
+
+ PyMODINIT_FUNC
+ PyInit_spam(void)
+ {
+ return PyModule_Create(&spam_module);
+ }
+
+
.. c:macro:: Py_ABS(x)
Return the absolute value of ``x``.
@@ -261,7 +285,7 @@ complete listing.
Objects, Types and Reference Counts
===================================
-.. index:: object: type
+.. index:: pair: object; type
Most Python/C API functions have one or more arguments as well as a return value
of type :c:expr:`PyObject*`. This type is a pointer to an opaque data type
@@ -287,52 +311,58 @@ true if (and only if) the object pointed to by *a* is a Python list.
Reference Counts
----------------
-The reference count is important because today's computers have a finite (and
-often severely limited) memory size; it counts how many different places there
-are that have a reference to an object. Such a place could be another object,
-or a global (or static) C variable, or a local variable in some C function.
-When an object's reference count becomes zero, the object is deallocated. If
-it contains references to other objects, their reference count is decremented.
-Those other objects may be deallocated in turn, if this decrement makes their
-reference count become zero, and so on. (There's an obvious problem with
-objects that reference each other here; for now, the solution is "don't do
-that.")
+The reference count is important because today's computers have a finite
+(and often severely limited) memory size; it counts how many different
+places there are that have a :term:`strong reference` to an object.
+Such a place could be another object, or a global (or static) C variable,
+or a local variable in some C function.
+When the last :term:`strong reference` to an object is released
+(i.e. its reference count becomes zero), the object is deallocated.
+If it contains references to other objects, those references are released.
+Those other objects may be deallocated in turn, if there are no more
+references to them, and so on. (There's an obvious problem with
+objects that reference each other here; for now, the solution
+is "don't do that.")
.. index::
single: Py_INCREF()
single: Py_DECREF()
-Reference counts are always manipulated explicitly. The normal way is to use
-the macro :c:func:`Py_INCREF` to increment an object's reference count by one,
-and :c:func:`Py_DECREF` to decrement it by one. The :c:func:`Py_DECREF` macro
+Reference counts are always manipulated explicitly. The normal way is
+to use the macro :c:func:`Py_INCREF` to take a new reference to an
+object (i.e. increment its reference count by one),
+and :c:func:`Py_DECREF` to release that reference (i.e. decrement the
+reference count by one). The :c:func:`Py_DECREF` macro
is considerably more complex than the incref one, since it must check whether
the reference count becomes zero and then cause the object's deallocator to be
-called. The deallocator is a function pointer contained in the object's type
-structure. The type-specific deallocator takes care of decrementing the
-reference counts for other objects contained in the object if this is a compound
+called. The deallocator is a function pointer contained in the object's type
+structure. The type-specific deallocator takes care of releasing references
+for other objects contained in the object if this is a compound
object type, such as a list, as well as performing any additional finalization
that's needed. There's no chance that the reference count can overflow; at
least as many bits are used to hold the reference count as there are distinct
memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``).
Thus, the reference count increment is a simple operation.
-It is not necessary to increment an object's reference count for every local
-variable that contains a pointer to an object. In theory, the object's
+It is not necessary to hold a :term:`strong reference` (i.e. increment
+the reference count) for every local variable that contains a pointer
+to an object. In theory, the object's
reference count goes up by one when the variable is made to point to it and it
goes down by one when the variable goes out of scope. However, these two
cancel each other out, so at the end the reference count hasn't changed. The
only real reason to use the reference count is to prevent the object from being
deallocated as long as our variable is pointing to it. If we know that there
is at least one other reference to the object that lives at least as long as
-our variable, there is no need to increment the reference count temporarily.
+our variable, there is no need to take a new :term:`strong reference`
+(i.e. increment the reference count) temporarily.
An important situation where this arises is in objects that are passed as
arguments to C functions in an extension module that are called from Python;
the call mechanism guarantees to hold a reference to every argument for the
duration of the call.
However, a common pitfall is to extract an object from a list and hold on to it
-for a while without incrementing its reference count. Some other operation might
-conceivably remove the object from the list, decrementing its reference count
+for a while without taking a new reference. Some other operation might
+conceivably remove the object from the list, releasing that reference,
and possibly deallocating it. The real danger is that innocent-looking
operations may invoke arbitrary Python code which could do this; there is a code
path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so
@@ -340,7 +370,8 @@ almost any operation is potentially dangerous.
A safe approach is to always use the generic operations (functions whose name
begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``).
-These operations always increment the reference count of the object they return.
+These operations always create a new :term:`strong reference`
+(i.e. increment the reference count) of the object they return.
This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when
they are done with the result; this soon becomes second nature.
@@ -356,7 +387,7 @@ to objects (objects are not owned: they are always shared). "Owning a
reference" means being responsible for calling Py_DECREF on it when the
reference is no longer needed. Ownership can also be transferred, meaning that
the code that receives ownership of the reference then becomes responsible for
-eventually decref'ing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF`
+eventually releasing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF`
when it's no longer needed---or passing on this responsibility (usually to its
caller). When a function passes ownership of a reference on to its caller, the
caller is said to receive a *new* reference. When no ownership is transferred,
@@ -414,9 +445,9 @@ For example, the above two blocks of code could be replaced by the following
It is much more common to use :c:func:`PyObject_SetItem` and friends with items
whose references you are only borrowing, like arguments that were passed in to
-the function you are writing. In that case, their behaviour regarding reference
-counts is much saner, since you don't have to increment a reference count so you
-can give a reference away ("have it be stolen"). For example, this function
+the function you are writing. In that case, their behaviour regarding references
+is much saner, since you don't have to take a new reference just so you
+can give that reference away ("have it be stolen"). For example, this function
sets all items of a list (actually, any mutable sequence) to a given item::
int
@@ -616,7 +647,7 @@ and lose important information about the exact cause of the error.
.. index:: single: sum_sequence()
A simple example of detecting exceptions and passing them on is shown in the
-:c:func:`sum_sequence` example above. It so happens that this example doesn't
+:c:func:`!sum_sequence` example above. It so happens that this example doesn't
need to clean up any owned references when it detects an error. The following
example function shows some error cleanup. First, to remind you why you like
Python, we show the equivalent Python code::
@@ -705,9 +736,9 @@ interpreter can only be used after the interpreter has been initialized.
.. index::
single: Py_Initialize()
- module: builtins
- module: __main__
- module: sys
+ pair: module; builtins
+ pair: module; __main__
+ pair: module; sys
triple: module; search; path
single: path (in module sys)
@@ -739,14 +770,14 @@ environment variable :envvar:`PYTHONHOME`, or insert additional directories in
front of the standard path by setting :envvar:`PYTHONPATH`.
.. index::
- single: Py_SetProgramName()
single: Py_GetPath()
single: Py_GetPrefix()
single: Py_GetExecPrefix()
single: Py_GetProgramFullPath()
-The embedding application can steer the search by calling
-``Py_SetProgramName(file)`` *before* calling :c:func:`Py_Initialize`. Note that
+The embedding application can steer the search by setting
+:c:member:`PyConfig.program_name` *before* calling
+:c:func:`Py_InitializeFromConfig`. Note that
:envvar:`PYTHONHOME` still overrides this and :envvar:`PYTHONPATH` is still
inserted in front of the standard path. An application that requires total
control has to provide its own implementation of :c:func:`Py_GetPath`,
diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst
index 3fcf099134d4dd4..6b7ba8c99791634 100644
--- a/Doc/c-api/iterator.rst
+++ b/Doc/c-api/iterator.rst
@@ -6,7 +6,7 @@ Iterator Objects
----------------
Python provides two general-purpose iterator objects. The first, a sequence
-iterator, works with an arbitrary sequence supporting the :meth:`__getitem__`
+iterator, works with an arbitrary sequence supporting the :meth:`~object.__getitem__`
method. The second works with a callable object and a sentinel value, calling
the callable for each item in the sequence, and ending the iteration when the
sentinel value is returned.
@@ -19,7 +19,7 @@ sentinel value is returned.
types.
-.. c:function:: int PySeqIter_Check(op)
+.. c:function:: int PySeqIter_Check(PyObject *op)
Return true if the type of *op* is :c:data:`PySeqIter_Type`. This function
always succeeds.
@@ -38,7 +38,7 @@ sentinel value is returned.
two-argument form of the :func:`iter` built-in function.
-.. c:function:: int PyCallIter_Check(op)
+.. c:function:: int PyCallIter_Check(PyObject *op)
Return true if the type of *op* is :c:data:`PyCallIter_Type`. This
function always succeeds.
diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst
index f9e65354a259f48..c8b64bad702f508 100644
--- a/Doc/c-api/list.rst
+++ b/Doc/c-api/list.rst
@@ -5,7 +5,7 @@
List Objects
------------
-.. index:: object: list
+.. index:: pair: object; list
.. c:type:: PyListObject
@@ -45,7 +45,7 @@ List Objects
.. c:function:: Py_ssize_t PyList_Size(PyObject *list)
- .. index:: builtin: len
+ .. index:: pair: built-in function; len
Return the length of the list object in *list*; this is equivalent to
``len(list)`` on a list object.
@@ -86,6 +86,10 @@ List Objects
Macro form of :c:func:`PyList_SetItem` without error checking. This is
normally only used to fill in new lists where there is no previous content.
+ Bounds checking is performed as an assertion if Python is built in
+ :ref:`debug mode ` or :option:`with assertions
+ <--with-assertions>`.
+
.. note::
This macro "steals" a reference to *item*, and, unlike
@@ -124,6 +128,30 @@ List Objects
list is not supported.
+.. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable)
+
+ Extend *list* with the contents of *iterable*. This is the same as
+ ``PyList_SetSlice(list, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, iterable)``
+ and analogous to ``list.extend(iterable)`` or ``list += iterable``.
+
+ Raise an exception and return ``-1`` if *list* is not a :class:`list`
+ object. Return 0 on success.
+
+ .. versionadded:: 3.13
+
+
+.. c:function:: int PyList_Clear(PyObject *list)
+
+ Remove all items from *list*. This is the same as
+ ``PyList_SetSlice(list, 0, PY_SSIZE_T_MAX, NULL)`` and analogous to
+ ``list.clear()`` or ``del list[:]``.
+
+ Raise an exception and return ``-1`` if *list* is not a :class:`list`
+ object. Return 0 on success.
+
+ .. versionadded:: 3.13
+
+
.. c:function:: int PyList_Sort(PyObject *list)
Sort the items of *list* in place. Return ``0`` on success, ``-1`` on
@@ -138,7 +166,7 @@ List Objects
.. c:function:: PyObject* PyList_AsTuple(PyObject *list)
- .. index:: builtin: tuple
+ .. index:: pair: built-in function; tuple
Return a new tuple object containing the contents of *list*; equivalent to
``tuple(list)``.
diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index 4f6f865db8be13c..045604870d3c840 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -5,8 +5,8 @@
Integer Objects
---------------
-.. index:: object: long integer
- object: integer
+.. index:: pair: object; long integer
+ pair: object; integer
All integers are implemented as "long" integer objects of arbitrary size.
@@ -94,6 +94,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
ignored. If there are no digits or *str* is not NULL-terminated following the
digits and trailing whitespace, :exc:`ValueError` will be raised.
+ .. seealso:: Python methods :meth:`int.to_bytes` and :meth:`int.from_bytes`
+ to convert a :c:type:`PyLongObject` to/from an array of bytes in base
+ ``256``. You can call those from C using :c:func:`PyObject_CallMethod`.
+
.. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base)
@@ -117,7 +121,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
single: OverflowError (built-in exception)
Return a C :c:expr:`long` representation of *obj*. If *obj* is not an
- instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method
+ instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method
(if present) to convert it to a :c:type:`PyLongObject`.
Raise :exc:`OverflowError` if the value of *obj* is out of range for a
@@ -126,30 +130,38 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. versionchanged:: 3.8
- Use :meth:`__index__` if available.
+ Use :meth:`~object.__index__` if available.
.. versionchanged:: 3.10
- This function will no longer use :meth:`__int__`.
+ This function will no longer use :meth:`~object.__int__`.
+
+
+.. c:function:: int PyLong_AsInt(PyObject *obj)
+
+ Similar to :c:func:`PyLong_AsLong`, but store the result in a C
+ :c:expr:`int` instead of a C :c:expr:`long`.
+
+ .. versionadded:: 3.13
.. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow)
Return a C :c:expr:`long` representation of *obj*. If *obj* is not an
- instance of :c:type:`PyLongObject`, first call its :meth:`__index__`
+ instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__`
method (if present) to convert it to a :c:type:`PyLongObject`.
- If the value of *obj* is greater than :const:`LONG_MAX` or less than
- :const:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and
+ If the value of *obj* is greater than :c:macro:`LONG_MAX` or less than
+ :c:macro:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and
return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception
occurs set *\*overflow* to ``0`` and return ``-1`` as usual.
Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. versionchanged:: 3.8
- Use :meth:`__index__` if available.
+ Use :meth:`~object.__index__` if available.
.. versionchanged:: 3.10
- This function will no longer use :meth:`__int__`.
+ This function will no longer use :meth:`~object.__int__`.
.. c:function:: long long PyLong_AsLongLong(PyObject *obj)
@@ -158,7 +170,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
single: OverflowError (built-in exception)
Return a C :c:expr:`long long` representation of *obj*. If *obj* is not an
- instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method
+ instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method
(if present) to convert it to a :c:type:`PyLongObject`.
Raise :exc:`OverflowError` if the value of *obj* is out of range for a
@@ -167,20 +179,20 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
.. versionchanged:: 3.8
- Use :meth:`__index__` if available.
+ Use :meth:`~object.__index__` if available.
.. versionchanged:: 3.10
- This function will no longer use :meth:`__int__`.
+ This function will no longer use :meth:`~object.__int__`.
.. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow)
Return a C :c:expr:`long long` representation of *obj*. If *obj* is not an
- instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method
+ instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method
(if present) to convert it to a :c:type:`PyLongObject`.
- If the value of *obj* is greater than :const:`LLONG_MAX` or less than
- :const:`LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively,
+ If the value of *obj* is greater than :c:macro:`LLONG_MAX` or less than
+ :c:macro:`LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively,
and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other
exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual.
@@ -189,10 +201,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
.. versionadded:: 3.2
.. versionchanged:: 3.8
- Use :meth:`__index__` if available.
+ Use :meth:`~object.__index__` if available.
.. versionchanged:: 3.10
- This function will no longer use :meth:`__int__`.
+ This function will no longer use :meth:`~object.__int__`.
.. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong)
@@ -263,7 +275,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
.. c:function:: unsigned long PyLong_AsUnsignedLongMask(PyObject *obj)
Return a C :c:expr:`unsigned long` representation of *obj*. If *obj* is not
- an instance of :c:type:`PyLongObject`, first call its :meth:`__index__`
+ an instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__`
method (if present) to convert it to a :c:type:`PyLongObject`.
If the value of *obj* is out of range for an :c:expr:`unsigned long`,
@@ -273,17 +285,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
disambiguate.
.. versionchanged:: 3.8
- Use :meth:`__index__` if available.
+ Use :meth:`~object.__index__` if available.
.. versionchanged:: 3.10
- This function will no longer use :meth:`__int__`.
+ This function will no longer use :meth:`~object.__int__`.
.. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj)
Return a C :c:expr:`unsigned long long` representation of *obj*. If *obj*
is not an instance of :c:type:`PyLongObject`, first call its
- :meth:`__index__` method (if present) to convert it to a
+ :meth:`~object.__index__` method (if present) to convert it to a
:c:type:`PyLongObject`.
If the value of *obj* is out of range for an :c:expr:`unsigned long long`,
@@ -293,10 +305,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
to disambiguate.
.. versionchanged:: 3.8
- Use :meth:`__index__` if available.
+ Use :meth:`~object.__index__` if available.
.. versionchanged:: 3.10
- This function will no longer use :meth:`__int__`.
+ This function will no longer use :meth:`~object.__int__`.
.. c:function:: double PyLong_AsDouble(PyObject *pylong)
@@ -318,3 +330,27 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
with :c:func:`PyLong_FromVoidPtr`.
Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+
+
+.. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op)
+
+ Return 1 if *op* is compact, 0 otherwise.
+
+ This function makes it possible for performance-critical code to implement
+ a “fast path†for small integers. For compact values use
+ :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a
+ :c:func:`PyLong_As* ` function or
+ :c:func:`calling ` :meth:`int.to_bytes`.
+
+ The speedup is expected to be negligible for most users.
+
+ Exactly what values are considered compact is an implementation detail
+ and is subject to change.
+
+.. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op)
+
+ If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`,
+ return its value.
+
+ Otherwise, the return value is undefined.
+
diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst
index 3c9d282c6d0ab02..1f55c0aa955c75b 100644
--- a/Doc/c-api/mapping.rst
+++ b/Doc/c-api/mapping.rst
@@ -13,14 +13,14 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and
Return ``1`` if the object provides the mapping protocol or supports slicing,
and ``0`` otherwise. Note that it returns ``1`` for Python classes with
- a :meth:`__getitem__` method, since in general it is impossible to
+ a :meth:`~object.__getitem__` method, since in general it is impossible to
determine what type of keys the class supports. This function always succeeds.
.. c:function:: Py_ssize_t PyMapping_Size(PyObject *o)
Py_ssize_t PyMapping_Length(PyObject *o)
- .. index:: builtin: len
+ .. index:: pair: built-in function; len
Returns the number of keys in object *o* on success, and ``-1`` on failure.
This is equivalent to the Python expression ``len(o)``.
@@ -28,52 +28,100 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and
.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key)
- Return element of *o* corresponding to the string *key* or ``NULL`` on failure.
- This is the equivalent of the Python expression ``o[key]``.
- See also :c:func:`PyObject_GetItem`.
+ This is the same as :c:func:`PyObject_GetItem`, but *key* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
+
+
+.. c:function:: int PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result)
+
+ Variant of :c:func:`PyObject_GetItem` which doesn't raise
+ :exc:`KeyError` if the key is not found.
+
+ If the key is found, return ``1`` and set *\*result* to a new
+ :term:`strong reference` to the corresponding value.
+ If the key is not found, return ``0`` and set *\*result* to ``NULL``;
+ the :exc:`KeyError` is silenced.
+ If an error other than :exc:`KeyError` is raised, return ``-1`` and
+ set *\*result* to ``NULL``.
+
+ .. versionadded:: 3.13
+
+
+.. c:function:: int PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result)
+
+ This is the same as :c:func:`PyMapping_GetOptionalItem`, but *key* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
+
+ .. versionadded:: 3.13
.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v)
- Map the string *key* to the value *v* in object *o*. Returns ``-1`` on
- failure. This is the equivalent of the Python statement ``o[key] = v``.
- See also :c:func:`PyObject_SetItem`. This function *does not* steal a
- reference to *v*.
+ This is the same as :c:func:`PyObject_SetItem`, but *key* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
.. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key)
- Remove the mapping for the object *key* from the object *o*. Return ``-1``
- on failure. This is equivalent to the Python statement ``del o[key]``.
This is an alias of :c:func:`PyObject_DelItem`.
.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key)
- Remove the mapping for the string *key* from the object *o*. Return ``-1``
- on failure. This is equivalent to the Python statement ``del o[key]``.
+ This is the same as :c:func:`PyObject_DelItem`, but *key* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
-.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key)
+.. c:function:: int PyMapping_HasKeyWithError(PyObject *o, PyObject *key)
Return ``1`` if the mapping object has the key *key* and ``0`` otherwise.
This is equivalent to the Python expression ``key in o``.
- This function always succeeds.
+ On failure, return ``-1``.
- Note that exceptions which occur while calling the :meth:`__getitem__`
- method will get suppressed.
- To get error reporting use :c:func:`PyObject_GetItem()` instead.
+ .. versionadded:: 3.13
-.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key)
+.. c:function:: int PyMapping_HasKeyStringWithError(PyObject *o, const char *key)
+
+ This is the same as :c:func:`PyMapping_HasKeyWithError`, but *key* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
+
+ .. versionadded:: 3.13
+
+
+.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key)
Return ``1`` if the mapping object has the key *key* and ``0`` otherwise.
This is equivalent to the Python expression ``key in o``.
This function always succeeds.
- Note that exceptions which occur while calling the :meth:`__getitem__`
- method and creating a temporary string object will get suppressed.
- To get error reporting use :c:func:`PyMapping_GetItemString()` instead.
+ .. note::
+
+ Exceptions which occur when this calls :meth:`~object.__getitem__`
+ method are silently ignored.
+ For proper error handling, use :c:func:`PyMapping_HasKeyWithError`,
+ :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead.
+
+
+.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key)
+
+ This is the same as :c:func:`PyMapping_HasKey`, but *key* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
+
+ .. note::
+
+ Exceptions that occur when this calls :meth:`~object.__getitem__`
+ method or while creating the temporary :class:`str`
+ object are silently ignored.
+ For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`,
+ :c:func:`PyMapping_GetOptionalItemString` or
+ :c:func:`PyMapping_GetItemString` instead.
.. c:function:: PyObject* PyMapping_Keys(PyObject *o)
diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst
index 8e25968c6909fd6..489f1580a414b2d 100644
--- a/Doc/c-api/marshal.rst
+++ b/Doc/c-api/marshal.rst
@@ -25,12 +25,16 @@ unmarshalling. Version 2 uses a binary format for floating point numbers.
the least-significant 32 bits of *value*; regardless of the size of the
native :c:expr:`long` type. *version* indicates the file format.
+ This function can fail, in which case it sets the error indicator.
+ Use :c:func:`PyErr_Occurred` to check for that.
.. c:function:: void PyMarshal_WriteObjectToFile(PyObject *value, FILE *file, int version)
Marshal a Python object, *value*, to *file*.
*version* indicates the file format.
+ This function can fail, in which case it sets the error indicator.
+ Use :c:func:`PyErr_Occurred` to check for that.
.. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version)
diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
index 7041c15d23fb839..1f392e55078e775 100644
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -136,7 +136,7 @@ need to be held.
The :ref:`default raw memory allocator ` uses
the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc`
-and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting
+and :c:func:`!free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting
zero bytes.
.. versionadded:: 3.4
@@ -264,14 +264,14 @@ The following type-oriented macros are provided for convenience. Note that
*TYPE* refers to any C type.
-.. c:function:: TYPE* PyMem_New(TYPE, size_t n)
+.. c:macro:: PyMem_New(TYPE, n)
Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of
memory. Returns a pointer cast to :c:expr:`TYPE*`. The memory will not have
been initialized in any way.
-.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n)
+.. c:macro:: PyMem_Resize(p, TYPE, n)
Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n *
sizeof(TYPE))`` bytes. Returns a pointer cast to :c:expr:`TYPE*`. On return,
@@ -391,6 +391,8 @@ Legend:
* ``malloc``: system allocators from the standard C library, C functions:
:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`.
* ``pymalloc``: :ref:`pymalloc memory allocator `.
+* ``mimalloc``: :ref:`mimalloc memory allocator `. The pymalloc
+ allocator will be used if mimalloc support isn't available.
* "+ debug": with :ref:`debug hooks on the Python memory allocators
`.
* "Debug build": :ref:`Python build in debug mode `.
@@ -423,7 +425,7 @@ Customize Memory Allocators
+----------------------------------------------------------+---------------------------------------+
.. versionchanged:: 3.5
- The :c:type:`PyMemAllocator` structure was renamed to
+ The :c:type:`!PyMemAllocator` structure was renamed to
:c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added.
@@ -431,6 +433,8 @@ Customize Memory Allocators
Enum used to identify an allocator domain. Domains:
+ .. c:namespace:: NULL
+
.. c:macro:: PYMEM_DOMAIN_RAW
Functions:
@@ -470,10 +474,14 @@ Customize Memory Allocators
The new allocator must return a distinct non-``NULL`` pointer when requesting
zero bytes.
- For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be
+ For the :c:macro:`PYMEM_DOMAIN_RAW` domain, the allocator must be
thread-safe: the :term:`GIL ` is not held when the
allocator is called.
+ For the remaining domains, the allocator must also be thread-safe:
+ the allocator may be called in different interpreters that do not
+ share a ``GIL``.
+
If the new allocator is not a hook (does not call the previous allocator),
the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the
debug hooks on top on the new allocator.
@@ -485,19 +493,21 @@ Customize Memory Allocators
:c:func:`PyMem_SetAllocator` does have the following contract:
- * It can be called after :c:func:`Py_PreInitialize` and before
- :c:func:`Py_InitializeFromConfig` to install a custom memory
- allocator. There are no restrictions over the installed allocator
- other than the ones imposed by the domain (for instance, the Raw
- Domain allows the allocator to be called without the GIL held). See
- :ref:`the section on allocator domains ` for more
- information.
+ * It can be called after :c:func:`Py_PreInitialize` and before
+ :c:func:`Py_InitializeFromConfig` to install a custom memory
+ allocator. There are no restrictions over the installed allocator
+ other than the ones imposed by the domain (for instance, the Raw
+ Domain allows the allocator to be called without the GIL held). See
+ :ref:`the section on allocator domains ` for more
+ information.
- * If called after Python has finish initializing (after
- :c:func:`Py_InitializeFromConfig` has been called) the allocator
- **must** wrap the existing allocator. Substituting the current
- allocator for some other arbitrary one is **not supported**.
+ * If called after Python has finish initializing (after
+ :c:func:`Py_InitializeFromConfig` has been called) the allocator
+ **must** wrap the existing allocator. Substituting the current
+ allocator for some other arbitrary one is **not supported**.
+ .. versionchanged:: 3.12
+ All allocators must be thread-safe.
.. c:function:: void PyMem_SetupDebugHooks(void)
@@ -536,8 +546,8 @@ Runtime checks:
- Detect write before the start of the buffer (buffer underflow).
- Detect write after the end of the buffer (buffer overflow).
- Check that the :term:`GIL ` is held when
- allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex:
- :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex:
+ allocator functions of :c:macro:`PYMEM_DOMAIN_OBJ` (ex:
+ :c:func:`PyObject_Malloc`) and :c:macro:`PYMEM_DOMAIN_MEM` (ex:
:c:func:`PyMem_Malloc`) domains are called.
On error, the debug hooks use the :mod:`tracemalloc` module to get the
@@ -557,9 +567,9 @@ that the treatment of negative indices differs from a Python slice):
``p[-S]``
API identifier (ASCII character):
- * ``'r'`` for :c:data:`PYMEM_DOMAIN_RAW`.
- * ``'m'`` for :c:data:`PYMEM_DOMAIN_MEM`.
- * ``'o'`` for :c:data:`PYMEM_DOMAIN_OBJ`.
+ * ``'r'`` for :c:macro:`PYMEM_DOMAIN_RAW`.
+ * ``'m'`` for :c:macro:`PYMEM_DOMAIN_MEM`.
+ * ``'o'`` for :c:macro:`PYMEM_DOMAIN_OBJ`.
``p[-S+1:0]``
Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads.
@@ -581,7 +591,7 @@ that the treatment of negative indices differs from a Python slice):
default).
A serial number, incremented by 1 on each call to a malloc-like or
- realloc-like function. Big-endian ``size_t``. If "bad memory" is detected
+ realloc-like function. Big-endian :c:type:`size_t`. If "bad memory" is detected
later, the serial number gives an excellent way to set a breakpoint on the
next run, to capture the instant at which this block was passed out. The
static function bumpserialno() in obmalloc.c is the only place the serial
@@ -601,7 +611,7 @@ PYMEM_CLEANBYTE (meaning uninitialized memory is getting used).
compiled in release mode. On error, the debug hooks now use
:mod:`tracemalloc` to get the traceback where a memory block was allocated.
The debug hooks now also check if the GIL is held when functions of
- :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are
+ :c:macro:`PYMEM_DOMAIN_OBJ` and :c:macro:`PYMEM_DOMAIN_MEM` domains are
called.
.. versionchanged:: 3.8
@@ -618,17 +628,18 @@ The pymalloc allocator
Python has a *pymalloc* allocator optimized for small objects (smaller or equal
to 512 bytes) with a short lifetime. It uses memory mappings called "arenas"
-with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and
+with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit
+platforms. It falls back to :c:func:`PyMem_RawMalloc` and
:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes.
*pymalloc* is the :ref:`default allocator ` of the
-:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and
-:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains.
+:c:macro:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and
+:c:macro:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains.
The arena allocator uses the following functions:
-* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows,
-* :c:func:`mmap` and :c:func:`munmap` if available,
+* :c:func:`!VirtualAlloc` and :c:func:`!VirtualFree` on Windows,
+* :c:func:`!mmap` and :c:func:`!munmap` if available,
* :c:func:`malloc` and :c:func:`free` otherwise.
This allocator is disabled if Python is configured with the
@@ -663,6 +674,16 @@ Customize pymalloc Arena Allocator
Set the arena allocator.
+.. _mimalloc:
+
+The mimalloc allocator
+======================
+
+.. versionadded:: 3.13
+
+Python supports the mimalloc allocator when the underlying platform support is available.
+mimalloc "is a general purpose allocator with excellent performance characteristics.
+Initially developed by Daan Leijen for the runtime systems of the Koka and Lean languages."
tracemalloc C API
=================
@@ -732,8 +753,8 @@ allocators operating on different heaps. ::
free(buf1); /* Fatal -- should be PyMem_Del() */
In addition to the functions aimed at handling raw memory blocks from the Python
-heap, objects in Python are allocated and released with :c:func:`PyObject_New`,
-:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`.
+heap, objects in Python are allocated and released with :c:macro:`PyObject_New`,
+:c:macro:`PyObject_NewVar` and :c:func:`PyObject_Del`.
These will be explained in the next chapter on defining and implementing new
object types in C.
diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst
index ebd5c7760437bf3..2aa43318e7a455c 100644
--- a/Doc/c-api/memoryview.rst
+++ b/Doc/c-api/memoryview.rst
@@ -3,7 +3,7 @@
.. _memoryview-objects:
.. index::
- object: memoryview
+ pair: object; memoryview
MemoryView objects
------------------
diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst
index 6e7e1e21aa93f29..0d75ab8e1af1118 100644
--- a/Doc/c-api/method.rst
+++ b/Doc/c-api/method.rst
@@ -5,10 +5,10 @@
Instance Method Objects
-----------------------
-.. index:: object: instancemethod
+.. index:: pair: object; instancemethod
-An instance method is a wrapper for a :c:data:`PyCFunction` and the new way
-to bind a :c:data:`PyCFunction` to a class object. It replaces the former call
+An instance method is a wrapper for a :c:type:`PyCFunction` and the new way
+to bind a :c:type:`PyCFunction` to a class object. It replaces the former call
``PyMethod_New(func, NULL, class)``.
@@ -47,7 +47,7 @@ to bind a :c:data:`PyCFunction` to a class object. It replaces the former call
Method Objects
--------------
-.. index:: object: method
+.. index:: pair: object; method
Methods are bound function objects. Methods are always bound to an instance of
a user-defined class. Unbound methods (methods bound to a class object) are
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index e2ba157b32c7d99..979b22261efa3b3 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -5,7 +5,7 @@
Module Objects
--------------
-.. index:: object: module
+.. index:: pair: object; module
.. c:var:: PyTypeObject PyModule_Type
@@ -119,7 +119,7 @@ Module Objects
encoded to 'utf-8'.
.. deprecated:: 3.2
- :c:func:`PyModule_GetFilename` raises :c:type:`UnicodeEncodeError` on
+ :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on
unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead.
@@ -145,7 +145,7 @@ or request "multi-phase initialization" by returning the definition struct itsel
.. c:member:: PyModuleDef_Base m_base
- Always initialize this member to :const:`PyModuleDef_HEAD_INIT`.
+ Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`.
.. c:member:: const char *m_name
@@ -164,7 +164,7 @@ or request "multi-phase initialization" by returning the definition struct itsel
This memory area is allocated based on *m_size* on module creation,
and freed when the module object is deallocated, after the
- :c:member:`m_free` function has been called, if present.
+ :c:member:`~PyModuleDef.m_free` function has been called, if present.
Setting ``m_size`` to ``-1`` means that the module does not support
sub-interpreters, because it has global state.
@@ -202,7 +202,7 @@ or request "multi-phase initialization" by returning the definition struct itsel
This function is not called if the module state was requested but is not
allocated yet. This is the case immediately after the module is created
and before the module is executed (:c:data:`Py_mod_exec` function). More
- precisely, this function is not called if :c:member:`m_size` is greater
+ precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater
than 0 and the module state (as returned by :c:func:`PyModule_GetState`)
is ``NULL``.
@@ -217,7 +217,7 @@ or request "multi-phase initialization" by returning the definition struct itsel
This function is not called if the module state was requested but is not
allocated yet. This is the case immediately after the module is created
and before the module is executed (:c:data:`Py_mod_exec` function). More
- precisely, this function is not called if :c:member:`m_size` is greater
+ precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater
than 0 and the module state (as returned by :c:func:`PyModule_GetState`)
is ``NULL``.
@@ -238,7 +238,7 @@ or request "multi-phase initialization" by returning the definition struct itsel
This function is not called if the module state was requested but is not
allocated yet. This is the case immediately after the module is created
and before the module is executed (:c:data:`Py_mod_exec` function). More
- precisely, this function is not called if :c:member:`m_size` is greater
+ precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater
than 0 and the module state (as returned by :c:func:`PyModule_GetState`)
is ``NULL``.
@@ -256,7 +256,7 @@ of the following two module creation functions:
Create a new module object, given the definition in *def*. This behaves
like :c:func:`PyModule_Create2` with *module_api_version* set to
- :const:`PYTHON_API_VERSION`.
+ :c:macro:`PYTHON_API_VERSION`.
.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version)
@@ -282,7 +282,7 @@ An alternate way to specify extensions is to request "multi-phase initialization
Extension modules created this way behave more like Python modules: the
initialization is split between the *creation phase*, when the module object
is created, and the *execution phase*, when it is populated.
-The distinction is similar to the :py:meth:`__new__` and :py:meth:`__init__` methods
+The distinction is similar to the :py:meth:`!__new__` and :py:meth:`!__init__` methods
of classes.
Unlike modules created using single-phase initialization, these modules are not
@@ -293,7 +293,7 @@ By default, multiple modules created from the same definition should be
independent: changes to one should not affect the others.
This means that all state should be specific to the module object (using e.g.
using :c:func:`PyModule_GetState`), or its contents (such as the module's
-:attr:`__dict__` or individual classes created with :c:func:`PyType_FromSpec`).
+:attr:`~object.__dict__` or individual classes created with :c:func:`PyType_FromSpec`).
All modules created using multi-phase initialization are expected to support
:ref:`sub-interpreters `. Making sure multiple modules
@@ -338,6 +338,7 @@ The available slot types are:
The *value* pointer of this slot must point to a function of the signature:
.. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def)
+ :noindex:
The function receives a :py:class:`~importlib.machinery.ModuleSpec`
instance, as defined in :PEP:`451`, and the module definition.
@@ -372,10 +373,44 @@ The available slot types are:
The signature of the function is:
.. c:function:: int exec_module(PyObject* module)
+ :noindex:
If multiple ``Py_mod_exec`` slots are specified, they are processed in the
order they appear in the *m_slots* array.
+.. c:macro:: Py_mod_multiple_interpreters
+
+ Specifies one of the following values:
+
+ .. c:namespace:: NULL
+
+ .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED
+
+ The module does not support being imported in subinterpreters.
+
+ .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED
+
+ The module supports being imported in subinterpreters,
+ but only when they share the main interpreter's GIL.
+ (See :ref:`isolating-extensions-howto`.)
+
+ .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED
+
+ The module supports being imported in subinterpreters,
+ even when they have their own GIL.
+ (See :ref:`isolating-extensions-howto`.)
+
+ This slot determines whether or not importing this module
+ in a subinterpreter will fail.
+
+ Multiple ``Py_mod_multiple_interpreters`` slots may not be specified
+ in one module definition.
+
+ If ``Py_mod_multiple_interpreters`` is not specified, the import
+ machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED``.
+
+ .. versionadded:: 3.12
+
See :PEP:`489` for more details on multi-phase initialization.
Low-level module creation functions
@@ -388,15 +423,15 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and
.. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec)
- Create a new module object, given the definition in *module* and the
+ Create a new module object, given the definition in *def* and the
ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2`
- with *module_api_version* set to :const:`PYTHON_API_VERSION`.
+ with *module_api_version* set to :c:macro:`PYTHON_API_VERSION`.
.. versionadded:: 3.5
.. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version)
- Create a new module object, given the definition in *module* and the
+ Create a new module object, given the definition in *def* and the
ModuleSpec *spec*, assuming the API version *module_api_version*.
If that version does not match the version of the running interpreter,
a :exc:`RuntimeWarning` is emitted.
@@ -486,59 +521,56 @@ state:
.. versionadded:: 3.10
+.. c:function:: int PyModule_Add(PyObject *module, const char *name, PyObject *value)
+
+ Similar to :c:func:`PyModule_AddObjectRef`, but "steals" a reference
+ to *value*.
+ It can be called with a result of function that returns a new reference
+ without bothering to check its result or even saving it to a variable.
+
+ Example usage::
+
+ if (PyModule_Add(module, "spam", PyBytes_FromString(value)) < 0) {
+ goto error;
+ }
+
+ .. versionadded:: 3.13
+
+
.. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value)
Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to
*value* on success (if it returns ``0``).
- The new :c:func:`PyModule_AddObjectRef` function is recommended, since it is
+ The new :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef`
+ functions are recommended, since it is
easy to introduce reference leaks by misusing the
:c:func:`PyModule_AddObject` function.
.. note::
Unlike other functions that steal references, ``PyModule_AddObject()``
- only decrements the reference count of *value* **on success**.
+ only releases the reference to *value* **on success**.
This means that its return value must be checked, and calling code must
- :c:func:`Py_DECREF` *value* manually on error.
+ :c:func:`Py_XDECREF` *value* manually on error.
Example usage::
- static int
- add_spam(PyObject *module, int value)
- {
- PyObject *obj = PyLong_FromLong(value);
- if (obj == NULL) {
- return -1;
- }
- if (PyModule_AddObject(module, "spam", obj) < 0) {
- Py_DECREF(obj);
- return -1;
- }
- // PyModule_AddObject() stole a reference to obj:
- // Py_DECREF(obj) is not needed here
- return 0;
- }
-
- The example can also be written without checking explicitly if *obj* is
- ``NULL``::
+ PyObject *obj = PyBytes_FromString(value);
+ if (PyModule_AddObject(module, "spam", obj) < 0) {
+ // If 'obj' is not NULL and PyModule_AddObject() failed,
+ // 'obj' strong reference must be deleted with Py_XDECREF().
+ // If 'obj' is NULL, Py_XDECREF() does nothing.
+ Py_XDECREF(obj);
+ goto error;
+ }
+ // PyModule_AddObject() stole a reference to obj:
+ // Py_XDECREF(obj) is not needed here.
- static int
- add_spam(PyObject *module, int value)
- {
- PyObject *obj = PyLong_FromLong(value);
- if (PyModule_AddObject(module, "spam", obj) < 0) {
- Py_XDECREF(obj);
- return -1;
- }
- // PyModule_AddObject() stole a reference to obj:
- // Py_DECREF(obj) is not needed here
- return 0;
- }
+ .. deprecated:: 3.13
- Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in
- this case, since *obj* can be ``NULL``.
+ :c:func:`PyModule_AddObject` is :term:`soft deprecated`.
.. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value)
@@ -555,7 +587,7 @@ state:
``NULL``-terminated. Return ``-1`` on error, ``0`` on success.
-.. c:function:: int PyModule_AddIntMacro(PyObject *module, macro)
+.. c:macro:: PyModule_AddIntMacro(module, macro)
Add an int constant to *module*. The name and the value are taken from
*macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int
@@ -563,7 +595,7 @@ state:
Return ``-1`` on error, ``0`` on success.
-.. c:function:: int PyModule_AddStringMacro(PyObject *module, macro)
+.. c:macro:: PyModule_AddStringMacro(module, macro)
Add a string constant to *module*.
diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst
index 26d2b7aab201baf..f1941fc4bc4e850 100644
--- a/Doc/c-api/none.rst
+++ b/Doc/c-api/none.rst
@@ -5,22 +5,22 @@
The ``None`` Object
-------------------
-.. index:: object: None
+.. index:: pair: object; None
Note that the :c:type:`PyTypeObject` for ``None`` is not directly exposed in the
Python/C API. Since ``None`` is a singleton, testing for object identity (using
-``==`` in C) is sufficient. There is no :c:func:`PyNone_Check` function for the
+``==`` in C) is sufficient. There is no :c:func:`!PyNone_Check` function for the
same reason.
.. c:var:: PyObject* Py_None
- The Python ``None`` object, denoting lack of value. This object has no methods.
- It needs to be treated just like any other object with respect to reference
- counts.
+ The Python ``None`` object, denoting lack of value. This object has no methods
+ and is :term:`immortal`.
+ .. versionchanged:: 3.12
+ :c:data:`Py_None` is :term:`immortal`.
.. c:macro:: Py_RETURN_NONE
- Properly handle returning :c:data:`Py_None` from within a C function (that is,
- increment the reference count of ``None`` and return it.)
+ Return :c:data:`Py_None` from a function.
diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst
index 70b91f8c2d0ca17..13d3c5af956905a 100644
--- a/Doc/c-api/number.rst
+++ b/Doc/c-api/number.rst
@@ -64,7 +64,7 @@ Number Protocol
.. c:function:: PyObject* PyNumber_Divmod(PyObject *o1, PyObject *o2)
- .. index:: builtin: divmod
+ .. index:: pair: built-in function; divmod
See the built-in function :func:`divmod`. Returns ``NULL`` on failure. This is
the equivalent of the Python expression ``divmod(o1, o2)``.
@@ -72,7 +72,7 @@ Number Protocol
.. c:function:: PyObject* PyNumber_Power(PyObject *o1, PyObject *o2, PyObject *o3)
- .. index:: builtin: pow
+ .. index:: pair: built-in function; pow
See the built-in function :func:`pow`. Returns ``NULL`` on failure. This is the
equivalent of the Python expression ``pow(o1, o2, o3)``, where *o3* is optional.
@@ -94,7 +94,7 @@ Number Protocol
.. c:function:: PyObject* PyNumber_Absolute(PyObject *o)
- .. index:: builtin: abs
+ .. index:: pair: built-in function; abs
Returns the absolute value of *o*, or ``NULL`` on failure. This is the equivalent
of the Python expression ``abs(o)``.
@@ -192,7 +192,7 @@ Number Protocol
.. c:function:: PyObject* PyNumber_InPlacePower(PyObject *o1, PyObject *o2, PyObject *o3)
- .. index:: builtin: pow
+ .. index:: pair: built-in function; pow
See the built-in function :func:`pow`. Returns ``NULL`` on failure. The operation
is done *in-place* when *o1* supports it. This is the equivalent of the Python
@@ -238,7 +238,7 @@ Number Protocol
.. c:function:: PyObject* PyNumber_Long(PyObject *o)
- .. index:: builtin: int
+ .. index:: pair: built-in function; int
Returns the *o* converted to an integer object on success, or ``NULL`` on
failure. This is the equivalent of the Python expression ``int(o)``.
@@ -246,7 +246,7 @@ Number Protocol
.. c:function:: PyObject* PyNumber_Float(PyObject *o)
- .. index:: builtin: float
+ .. index:: pair: built-in function; float
Returns the *o* converted to a float object on success, or ``NULL`` on failure.
This is the equivalent of the Python expression ``float(o)``.
diff --git a/Doc/c-api/objbuffer.rst b/Doc/c-api/objbuffer.rst
deleted file mode 100644
index 6b82a642d7ee427..000000000000000
--- a/Doc/c-api/objbuffer.rst
+++ /dev/null
@@ -1,55 +0,0 @@
-.. highlight:: c
-
-Old Buffer Protocol
--------------------
-
-.. deprecated:: 3.0
-
-These functions were part of the "old buffer protocol" API in Python 2.
-In Python 3, this protocol doesn't exist anymore but the functions are still
-exposed to ease porting 2.x code. They act as a compatibility wrapper
-around the :ref:`new buffer protocol `, but they don't give
-you control over the lifetime of the resources acquired when a buffer is
-exported.
-
-Therefore, it is recommended that you call :c:func:`PyObject_GetBuffer`
-(or the ``y*`` or ``w*`` :ref:`format codes ` with the
-:c:func:`PyArg_ParseTuple` family of functions) to get a buffer view over
-an object, and :c:func:`PyBuffer_Release` when the buffer view can be released.
-
-
-.. c:function:: int PyObject_AsCharBuffer(PyObject *obj, const char **buffer, Py_ssize_t *buffer_len)
-
- Returns a pointer to a read-only memory location usable as character-based
- input. The *obj* argument must support the single-segment character buffer
- interface. On success, returns ``0``, sets *buffer* to the memory location
- and *buffer_len* to the buffer length. Returns ``-1`` and sets a
- :exc:`TypeError` on error.
-
-
-.. c:function:: int PyObject_AsReadBuffer(PyObject *obj, const void **buffer, Py_ssize_t *buffer_len)
-
- Returns a pointer to a read-only memory location containing arbitrary data.
- The *obj* argument must support the single-segment readable buffer
- interface. On success, returns ``0``, sets *buffer* to the memory location
- and *buffer_len* to the buffer length. Returns ``-1`` and sets a
- :exc:`TypeError` on error.
-
-
-.. c:function:: int PyObject_CheckReadBuffer(PyObject *o)
-
- Returns ``1`` if *o* supports the single-segment readable buffer interface.
- Otherwise returns ``0``. This function always succeeds.
-
- Note that this function tries to get and release a buffer, and exceptions
- which occur while calling corresponding functions will get suppressed.
- To get error reporting use :c:func:`PyObject_GetBuffer()` instead.
-
-
-.. c:function:: int PyObject_AsWriteBuffer(PyObject *obj, void **buffer, Py_ssize_t *buffer_len)
-
- Returns a pointer to a writable memory location. The *obj* argument must
- support the single-segment, character buffer interface. On success,
- returns ``0``, sets *buffer* to the memory location and *buffer_len* to the
- buffer length. Returns ``-1`` and sets a :exc:`TypeError` on error.
-
diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst
index 5a25a2b6c9d3dbb..a4e3e74861a3152 100644
--- a/Doc/c-api/object.rst
+++ b/Doc/c-api/object.rst
@@ -15,39 +15,64 @@ Object Protocol
.. c:macro:: Py_RETURN_NOTIMPLEMENTED
Properly handle returning :c:data:`Py_NotImplemented` from within a C
- function (that is, increment the reference count of NotImplemented and
- return it).
+ function (that is, create a new :term:`strong reference`
+ to NotImplemented and return it).
.. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags)
Print an object *o*, on file *fp*. Returns ``-1`` on error. The flags argument
is used to enable certain printing options. The only option currently supported
- is :const:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written
+ is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written
instead of the :func:`repr`.
+.. c:function:: int PyObject_HasAttrWithError(PyObject *o, const char *attr_name)
+
+ Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise.
+ This is equivalent to the Python expression ``hasattr(o, attr_name)``.
+ On failure, return ``-1``.
+
+ .. versionadded:: 3.13
+
+
+.. c:function:: int PyObject_HasAttrStringWithError(PyObject *o, const char *attr_name)
+
+ This is the same as :c:func:`PyObject_HasAttrWithError`, but *attr_name* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
+
+ .. versionadded:: 3.13
+
+
.. c:function:: int PyObject_HasAttr(PyObject *o, PyObject *attr_name)
Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This
is equivalent to the Python expression ``hasattr(o, attr_name)``. This function
always succeeds.
- Note that exceptions which occur while calling :meth:`__getattr__` and
- :meth:`__getattribute__` methods will get suppressed.
- To get error reporting use :c:func:`PyObject_GetAttr()` instead.
+ .. note::
+
+ Exceptions that occur when this calls :meth:`~object.__getattr__` and
+ :meth:`~object.__getattribute__` methods are silently ignored.
+ For proper error handling, use :c:func:`PyObject_HasAttrWithError`,
+ :c:func:`PyObject_GetOptionalAttr` or :c:func:`PyObject_GetAttr` instead.
.. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name)
- Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This
- is equivalent to the Python expression ``hasattr(o, attr_name)``. This function
- always succeeds.
+ This is the same as :c:func:`PyObject_HasAttr`, but *attr_name* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
- Note that exceptions which occur while calling :meth:`__getattr__` and
- :meth:`__getattribute__` methods and creating a temporary string object
- will get suppressed.
- To get error reporting use :c:func:`PyObject_GetAttrString()` instead.
+ .. note::
+
+ Exceptions that occur when this calls :meth:`~object.__getattr__` and
+ :meth:`~object.__getattribute__` methods or while creating the temporary
+ :class:`str` object are silently ignored.
+ For proper error handling, use :c:func:`PyObject_HasAttrStringWithError`,
+ :c:func:`PyObject_GetOptionalAttrString`
+ or :c:func:`PyObject_GetAttrString` instead.
.. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name)
@@ -56,14 +81,43 @@ Object Protocol
value on success, or ``NULL`` on failure. This is the equivalent of the Python
expression ``o.attr_name``.
+ If the missing attribute should not be treated as a failure, you can use
+ :c:func:`PyObject_GetOptionalAttr` instead.
+
.. c:function:: PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name)
- Retrieve an attribute named *attr_name* from object *o*. Returns the attribute
- value on success, or ``NULL`` on failure. This is the equivalent of the Python
- expression ``o.attr_name``.
+ This is the same as :c:func:`PyObject_GetAttr`, but *attr_name* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
+
+ If the missing attribute should not be treated as a failure, you can use
+ :c:func:`PyObject_GetOptionalAttrString` instead.
+
+
+.. c:function:: int PyObject_GetOptionalAttr(PyObject *obj, PyObject *attr_name, PyObject **result);
+
+ Variant of :c:func:`PyObject_GetAttr` which doesn't raise
+ :exc:`AttributeError` if the attribute is not found.
+
+ If the attribute is found, return ``1`` and set *\*result* to a new
+ :term:`strong reference` to the attribute.
+ If the attribute is not found, return ``0`` and set *\*result* to ``NULL``;
+ the :exc:`AttributeError` is silenced.
+ If an error other than :exc:`AttributeError` is raised, return ``-1`` and
+ set *\*result* to ``NULL``.
+
+ .. versionadded:: 3.13
+.. c:function:: int PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result);
+
+ This is the same as :c:func:`PyObject_GetOptionalAttr`, but *attr_name* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
+
+ .. versionadded:: 3.13
+
.. c:function:: PyObject* PyObject_GenericGetAttr(PyObject *o, PyObject *name)
Generic attribute getter function that is meant to be put into a type
@@ -88,10 +142,9 @@ Object Protocol
.. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v)
- Set the value of the attribute named *attr_name*, for object *o*, to the value
- *v*. Raise an exception and return ``-1`` on failure;
- return ``0`` on success. This is the equivalent of the Python statement
- ``o.attr_name = v``.
+ This is the same as :c:func:`PyObject_SetAttr`, but *attr_name* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
If *v* is ``NULL``, the attribute is deleted, but this feature is
deprecated in favour of using :c:func:`PyObject_DelAttrString`.
@@ -117,8 +170,9 @@ Object Protocol
.. c:function:: int PyObject_DelAttrString(PyObject *o, const char *attr_name)
- Delete attribute named *attr_name*, for object *o*. Returns ``-1`` on failure.
- This is the equivalent of the Python statement ``del o.attr_name``.
+ This is the same as :c:func:`PyObject_DelAttr`, but *attr_name* is
+ specified as a :c:expr:`const char*` UTF-8 encoded bytes string,
+ rather than a :c:expr:`PyObject*`.
.. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context)
@@ -158,8 +212,8 @@ Object Protocol
.. c:function:: PyObject* PyObject_RichCompare(PyObject *o1, PyObject *o2, int opid)
Compare the values of *o1* and *o2* using the operation specified by *opid*,
- which must be one of :const:`Py_LT`, :const:`Py_LE`, :const:`Py_EQ`,
- :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``,
+ which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`,
+ :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``,
``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. This is the equivalent of
the Python expression ``o1 op o2``, where ``op`` is the operator corresponding
to *opid*. Returns the value of the comparison on success, or ``NULL`` on failure.
@@ -168,8 +222,8 @@ Object Protocol
.. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid)
Compare the values of *o1* and *o2* using the operation specified by *opid*,
- which must be one of :const:`Py_LT`, :const:`Py_LE`, :const:`Py_EQ`,
- :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``,
+ which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`,
+ :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``,
``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error,
``0`` if the result is false, ``1`` otherwise. This is the equivalent of the
Python expression ``o1 op o2``, where ``op`` is the operator corresponding to
@@ -177,11 +231,20 @@ Object Protocol
.. note::
If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool`
- will always return ``1`` for :const:`Py_EQ` and ``0`` for :const:`Py_NE`.
+ will always return ``1`` for :c:macro:`Py_EQ` and ``0`` for :c:macro:`Py_NE`.
+
+.. c:function:: PyObject* PyObject_Format(PyObject *obj, PyObject *format_spec)
+
+ Format *obj* using *format_spec*. This is equivalent to the Python
+ expression ``format(obj, format_spec)``.
+
+ *format_spec* may be ``NULL``. In this case the call is equivalent
+ to ``format(obj)``.
+ Returns the formatted string on success, ``NULL`` on failure.
.. c:function:: PyObject* PyObject_Repr(PyObject *o)
- .. index:: builtin: repr
+ .. index:: pair: built-in function; repr
Compute a string representation of object *o*. Returns the string
representation on success, ``NULL`` on failure. This is the equivalent of the
@@ -193,7 +256,7 @@ Object Protocol
.. c:function:: PyObject* PyObject_ASCII(PyObject *o)
- .. index:: builtin: ascii
+ .. index:: pair: built-in function; ascii
As :c:func:`PyObject_Repr`, compute a string representation of object *o*, but
escape the non-ASCII characters in the string returned by
@@ -218,7 +281,7 @@ Object Protocol
.. c:function:: PyObject* PyObject_Bytes(PyObject *o)
- .. index:: builtin: bytes
+ .. index:: pair: built-in function; bytes
Compute a bytes representation of object *o*. ``NULL`` is returned on
failure and a bytes object on success. This is equivalent to the Python
@@ -243,7 +306,7 @@ Object Protocol
Normally only class objects, i.e. instances of :class:`type` or a derived
class, are considered classes. However, objects can override this by having
- a :attr:`__bases__` attribute (which must be a tuple of base classes).
+ a :attr:`~class.__bases__` attribute (which must be a tuple of base classes).
.. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls)
@@ -260,16 +323,16 @@ Object Protocol
is an instance of *cls* if its class is a subclass of *cls*.
An instance *inst* can override what is considered its class by having a
- :attr:`__class__` attribute.
+ :attr:`~instance.__class__` attribute.
An object *cls* can override if it is considered a class, and what its base
- classes are, by having a :attr:`__bases__` attribute (which must be a tuple
+ classes are, by having a :attr:`~class.__bases__` attribute (which must be a tuple
of base classes).
.. c:function:: Py_hash_t PyObject_Hash(PyObject *o)
- .. index:: builtin: hash
+ .. index:: pair: built-in function; hash
Compute and return the hash value of an object *o*. On failure, return ``-1``.
This is the equivalent of the Python expression ``hash(o)``.
@@ -281,7 +344,7 @@ Object Protocol
.. c:function:: Py_hash_t PyObject_HashNotImplemented(PyObject *o)
- Set a :exc:`TypeError` indicating that ``type(o)`` is not hashable and return ``-1``.
+ Set a :exc:`TypeError` indicating that ``type(o)`` is not :term:`hashable` and return ``-1``.
This function receives special treatment when stored in a ``tp_hash`` slot,
allowing a type to explicitly indicate to the interpreter that it is not
hashable.
@@ -303,15 +366,16 @@ Object Protocol
.. c:function:: PyObject* PyObject_Type(PyObject *o)
- .. index:: builtin: type
+ .. index:: pair: built-in function; type
When *o* is non-``NULL``, returns a type object corresponding to the object type
of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This
- is equivalent to the Python expression ``type(o)``. This function increments the
- reference count of the return value. There's really no reason to use this
+ is equivalent to the Python expression ``type(o)``.
+ This function creates a new :term:`strong reference` to the return value.
+ There's really no reason to use this
function instead of the :c:func:`Py_TYPE()` function, which returns a
- pointer of type :c:expr:`PyTypeObject*`, except when the incremented reference
- count is needed.
+ pointer of type :c:expr:`PyTypeObject*`, except when a new
+ :term:`strong reference` is needed.
.. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type)
@@ -323,7 +387,7 @@ Object Protocol
.. c:function:: Py_ssize_t PyObject_Size(PyObject *o)
Py_ssize_t PyObject_Length(PyObject *o)
- .. index:: builtin: len
+ .. index:: pair: built-in function; len
Return the length of object *o*. If the object *o* provides either the sequence
and mapping protocols, the sequence length is returned. On error, ``-1`` is
@@ -386,3 +450,60 @@ Object Protocol
returns ``NULL`` if the object cannot be iterated.
.. versionadded:: 3.10
+
+.. c:function:: void *PyObject_GetTypeData(PyObject *o, PyTypeObject *cls)
+
+ Get a pointer to subclass-specific data reserved for *cls*.
+
+ The object *o* must be an instance of *cls*, and *cls* must have been
+ created using negative :c:member:`PyType_Spec.basicsize`.
+ Python does not check this.
+
+ On error, set an exception and return ``NULL``.
+
+ .. versionadded:: 3.12
+
+.. c:function:: Py_ssize_t PyType_GetTypeDataSize(PyTypeObject *cls)
+
+ Return the size of the instance memory space reserved for *cls*, i.e. the size of the
+ memory :c:func:`PyObject_GetTypeData` returns.
+
+ This may be larger than requested using :c:member:`-PyType_Spec.basicsize `;
+ it is safe to use this larger size (e.g. with :c:func:`!memset`).
+
+ The type *cls* **must** have been created using
+ negative :c:member:`PyType_Spec.basicsize`.
+ Python does not check this.
+
+ On error, set an exception and return a negative value.
+
+ .. versionadded:: 3.12
+
+.. c:function:: void *PyObject_GetItemData(PyObject *o)
+
+ Get a pointer to per-item data for a class with
+ :c:macro:`Py_TPFLAGS_ITEMS_AT_END`.
+
+ On error, set an exception and return ``NULL``.
+ :py:exc:`TypeError` is raised if *o* does not have
+ :c:macro:`Py_TPFLAGS_ITEMS_AT_END` set.
+
+ .. versionadded:: 3.12
+
+.. c:function:: int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
+
+ Visit the managed dictionary of *obj*.
+
+ This function must only be called in a traverse function of the type which
+ has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set.
+
+ .. versionadded:: 3.13
+
+.. c:function:: void PyObject_ClearManagedDict(PyObject *obj)
+
+ Clear the managed dictionary of *obj*.
+
+ This function must only be called in a traverse function of the type which
+ has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set.
+
+ .. versionadded:: 3.13
diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst
new file mode 100644
index 000000000000000..3d44d2eb6bf41d6
--- /dev/null
+++ b/Doc/c-api/perfmaps.rst
@@ -0,0 +1,50 @@
+.. highlight:: c
+
+.. _perfmaps:
+
+Support for Perf Maps
+----------------------
+
+On supported platforms (as of this writing, only Linux), the runtime can take
+advantage of *perf map files* to make Python functions visible to an external
+profiling tool (such as `perf `_).
+A running process may create a file in the ``/tmp`` directory, which contains entries
+that can map a section of executable code to a name. This interface is described in the
+`documentation of the Linux Perf tool `_.
+
+In Python, these helper APIs can be used by libraries and features that rely
+on generating machine code on the fly.
+
+Note that holding the Global Interpreter Lock (GIL) is not required for these APIs.
+
+.. c:function:: int PyUnstable_PerfMapState_Init(void)
+
+ Open the ``/tmp/perf-$pid.map`` file, unless it's already opened, and create
+ a lock to ensure thread-safe writes to the file (provided the writes are
+ done through :c:func:`PyUnstable_WritePerfMapEntry`). Normally, there's no need
+ to call this explicitly; just use :c:func:`PyUnstable_WritePerfMapEntry`
+ and it will initialize the state on first call.
+
+ Returns ``0`` on success, ``-1`` on failure to create/open the perf map file,
+ or ``-2`` on failure to create a lock. Check ``errno`` for more information
+ about the cause of a failure.
+
+.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, unsigned int code_size, const char *entry_name)
+
+ Write one single entry to the ``/tmp/perf-$pid.map`` file. This function is
+ thread safe. Here is what an example entry looks like::
+
+ # address size name
+ 7f3529fcf759 b py::bar:/run/t.py
+
+ Will call :c:func:`PyUnstable_PerfMapState_Init` before writing the entry, if
+ the perf map file is not already opened. Returns ``0`` on success, or the
+ same error codes as :c:func:`PyUnstable_PerfMapState_Init` on failure.
+
+.. c:function:: void PyUnstable_PerfMapState_Fini(void)
+
+ Close the perf map file opened by :c:func:`PyUnstable_PerfMapState_Init`.
+ This is called by the runtime itself during interpreter shut-down. In
+ general, there shouldn't be a reason to explicitly call this, except to
+ handle specific scenarios such as forking.
diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst
index cd1f2ef7076836c..68119a27b18ec26 100644
--- a/Doc/c-api/refcounting.rst
+++ b/Doc/c-api/refcounting.rst
@@ -7,14 +7,20 @@
Reference Counting
******************
-The macros in this section are used for managing reference counts of Python
-objects.
+The functions and macros in this section are used for managing reference counts
+of Python objects.
.. c:function:: Py_ssize_t Py_REFCNT(PyObject *o)
Get the reference count of the Python object *o*.
+ Note that the returned value may not actually reflect how many
+ references to the object are actually held. For example, some
+ objects are :term:`immortal` and have a very high refcount that does not
+ reflect the actual number of references. Consequently, do not rely
+ on the returned value to be accurate, other than a value of 0 or 1.
+
Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count.
.. versionchanged:: 3.11
@@ -28,36 +34,53 @@ objects.
Set the object *o* reference counter to *refcnt*.
+ This function has no effect on :term:`immortal` objects.
+
.. versionadded:: 3.9
+ .. versionchanged:: 3.12
+ Immortal objects are not modified.
+
.. c:function:: void Py_INCREF(PyObject *o)
- Increment the reference count for object *o*.
+ Indicate taking a new :term:`strong reference` to object *o*,
+ indicating it is in use and should not be destroyed.
+
+ This function has no effect on :term:`immortal` objects.
This function is usually used to convert a :term:`borrowed reference` to a
:term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be
used to create a new :term:`strong reference`.
+ When done using the object, release is by calling :c:func:`Py_DECREF`.
+
The object must not be ``NULL``; if you aren't sure that it isn't
``NULL``, use :c:func:`Py_XINCREF`.
+ Do not expect this function to actually modify *o* in any way.
+ For at least `some objects `_,
+ this function has no effect.
+
+ .. versionchanged:: 3.12
+ Immortal objects are not modified.
+
.. c:function:: void Py_XINCREF(PyObject *o)
- Increment the reference count for object *o*. The object may be ``NULL``, in
- which case the macro has no effect.
+ Similar to :c:func:`Py_INCREF`, but the object *o* can be ``NULL``,
+ in which case this has no effect.
See also :c:func:`Py_XNewRef`.
.. c:function:: PyObject* Py_NewRef(PyObject *o)
- Create a new :term:`strong reference` to an object: increment the reference
- count of the object *o* and return the object *o*.
+ Create a new :term:`strong reference` to an object:
+ call :c:func:`Py_INCREF` on *o* and return the object *o*.
When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF`
- should be called on it to decrement the object reference count.
+ should be called on it to release the reference.
The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be
``NULL``.
@@ -87,9 +110,14 @@ objects.
.. c:function:: void Py_DECREF(PyObject *o)
- Decrement the reference count for object *o*.
+ Release a :term:`strong reference` to object *o*, indicating the
+ reference is no longer used.
- If the reference count reaches zero, the object's type's deallocation
+ This function has no effect on :term:`immortal` objects.
+
+ Once the last :term:`strong reference` is released
+ (i.e. the object's reference count reaches 0),
+ the object's type's deallocation
function (which must not be ``NULL``) is invoked.
This function is usually used to delete a :term:`strong reference` before
@@ -98,10 +126,14 @@ objects.
The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``,
use :c:func:`Py_XDECREF`.
+ Do not expect this function to actually modify *o* in any way.
+ For at least `some objects `_,
+ this function has no effect.
+
.. warning::
The deallocation function can cause arbitrary Python code to be invoked (e.g.
- when a class instance with a :meth:`__del__` method is deallocated). While
+ when a class instance with a :meth:`~object.__del__` method is deallocated). While
exceptions in such code are not propagated, the executed code has free access to
all Python global variables. This means that any object that is reachable from
a global variable should be in a consistent state before :c:func:`Py_DECREF` is
@@ -109,33 +141,82 @@ objects.
reference to the deleted object in a temporary variable, update the list data
structure, and then call :c:func:`Py_DECREF` for the temporary variable.
+ .. versionchanged:: 3.12
+ Immortal objects are not modified.
+
.. c:function:: void Py_XDECREF(PyObject *o)
- Decrement the reference count for object *o*. The object may be ``NULL``, in
- which case the macro has no effect; otherwise the effect is the same as for
- :c:func:`Py_DECREF`, and the same warning applies.
+ Similar to :c:func:`Py_DECREF`, but the object *o* can be ``NULL``,
+ in which case this has no effect.
+ The same warning from :c:func:`Py_DECREF` applies here as well.
.. c:function:: void Py_CLEAR(PyObject *o)
- Decrement the reference count for object *o*. The object may be ``NULL``, in
+ Release a :term:`strong reference` for object *o*.
+ The object may be ``NULL``, in
which case the macro has no effect; otherwise the effect is the same as for
:c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning
for :c:func:`Py_DECREF` does not apply with respect to the object passed because
the macro carefully uses a temporary variable and sets the argument to ``NULL``
- before decrementing its reference count.
+ before releasing the reference.
+
+ It is a good idea to use this macro whenever releasing a reference
+ to an object that might be traversed during garbage collection.
+
+ .. versionchanged:: 3.12
+ The macro argument is now only evaluated once. If the argument has side
+ effects, these are no longer duplicated.
- It is a good idea to use this macro whenever decrementing the reference
- count of an object that might be traversed during garbage collection.
.. c:function:: void Py_IncRef(PyObject *o)
- Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`.
+ Indicate taking a new :term:`strong reference` to object *o*.
+ A function version of :c:func:`Py_XINCREF`.
It can be used for runtime dynamic embedding of Python.
.. c:function:: void Py_DecRef(PyObject *o)
- Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`.
+ Release a :term:`strong reference` to object *o*.
+ A function version of :c:func:`Py_XDECREF`.
It can be used for runtime dynamic embedding of Python.
+
+
+.. c:macro:: Py_SETREF(dst, src)
+
+ Macro safely releasing a :term:`strong reference` to object *dst*
+ and setting *dst* to *src*.
+
+ As in case of :c:func:`Py_CLEAR`, "the obvious" code can be deadly::
+
+ Py_DECREF(dst);
+ dst = src;
+
+ The safe way is::
+
+ Py_SETREF(dst, src);
+
+ That arranges to set *dst* to *src* _before_ releasing the reference
+ to the old value of *dst*, so that any code triggered as a side-effect
+ of *dst* getting torn down no longer believes *dst* points
+ to a valid object.
+
+ .. versionadded:: 3.6
+
+ .. versionchanged:: 3.12
+ The macro arguments are now only evaluated once. If an argument has side
+ effects, these are no longer duplicated.
+
+
+.. c:macro:: Py_XSETREF(dst, src)
+
+ Variant of :c:macro:`Py_SETREF` macro that uses :c:func:`Py_XDECREF` instead
+ of :c:func:`Py_DECREF`.
+
+ .. versionadded:: 3.6
+
+ .. versionchanged:: 3.12
+ The macro arguments are now only evaluated once. If an argument has side
+ effects, these are no longer duplicated.
diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst
index c78d273f9f149f5..ce28839f5ba739d 100644
--- a/Doc/c-api/sequence.rst
+++ b/Doc/c-api/sequence.rst
@@ -9,7 +9,7 @@ Sequence Protocol
.. c:function:: int PySequence_Check(PyObject *o)
Return ``1`` if the object provides the sequence protocol, and ``0`` otherwise.
- Note that it returns ``1`` for Python classes with a :meth:`__getitem__`
+ Note that it returns ``1`` for Python classes with a :meth:`~object.__getitem__`
method, unless they are :class:`dict` subclasses, since in general it
is impossible to determine what type of keys the class supports. This
function always succeeds.
@@ -18,7 +18,7 @@ Sequence Protocol
.. c:function:: Py_ssize_t PySequence_Size(PyObject *o)
Py_ssize_t PySequence_Length(PyObject *o)
- .. index:: builtin: len
+ .. index:: pair: built-in function; len
Returns the number of objects in sequence *o* on success, and ``-1`` on
failure. This is equivalent to the Python expression ``len(o)``.
@@ -120,7 +120,7 @@ Sequence Protocol
.. c:function:: PyObject* PySequence_Tuple(PyObject *o)
- .. index:: builtin: tuple
+ .. index:: pair: built-in function; tuple
Return a tuple object with the same contents as the sequence or iterable *o*,
or ``NULL`` on failure. If *o* is a tuple, a new reference will be returned,
diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst
index f0d905bae8ae44c..cba823aa027bd6d 100644
--- a/Doc/c-api/set.rst
+++ b/Doc/c-api/set.rst
@@ -9,8 +9,8 @@ Set Objects
.. index::
- object: set
- object: frozenset
+ pair: object; set
+ pair: object; frozenset
This section details the public API for :class:`set` and :class:`frozenset`
objects. Any functionality not listed below is best accessed using either
@@ -107,10 +107,10 @@ or :class:`frozenset` or instances of their subtypes.
.. c:function:: Py_ssize_t PySet_Size(PyObject *anyset)
- .. index:: builtin: len
+ .. index:: pair: built-in function; len
Return the length of a :class:`set` or :class:`frozenset` object. Equivalent to
- ``len(anyset)``. Raises a :exc:`PyExc_SystemError` if *anyset* is not a
+ ``len(anyset)``. Raises a :exc:`SystemError` if *anyset* is not a
:class:`set`, :class:`frozenset`, or an instance of a subtype.
@@ -122,9 +122,9 @@ or :class:`frozenset` or instances of their subtypes.
.. c:function:: int PySet_Contains(PyObject *anyset, PyObject *key)
Return ``1`` if found, ``0`` if not found, and ``-1`` if an error is encountered. Unlike
- the Python :meth:`__contains__` method, this function does not automatically
+ the Python :meth:`~object.__contains__` method, this function does not automatically
convert unhashable sets into temporary frozensets. Raise a :exc:`TypeError` if
- the *key* is unhashable. Raise :exc:`PyExc_SystemError` if *anyset* is not a
+ the *key* is unhashable. Raise :exc:`SystemError` if *anyset* is not a
:class:`set`, :class:`frozenset`, or an instance of a subtype.
@@ -147,9 +147,9 @@ subtypes but not for instances of :class:`frozenset` or its subtypes.
Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an
error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a
- :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard`
+ :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard`
method, this function does not automatically convert unhashable sets into
- temporary frozensets. Raise :exc:`PyExc_SystemError` if *set* is not an
+ temporary frozensets. Raise :exc:`SystemError` if *set* is not an
instance of :class:`set` or its subtype.
@@ -163,4 +163,6 @@ subtypes but not for instances of :class:`frozenset` or its subtypes.
.. c:function:: int PySet_Clear(PyObject *set)
- Empty an existing set of all elements.
+ Empty an existing set of all elements. Return ``0`` on
+ success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of
+ :class:`set` or its subtype.
diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst
index 8271d9acfb645e1..27a1757c745d8bf 100644
--- a/Doc/c-api/slice.rst
+++ b/Doc/c-api/slice.rst
@@ -34,7 +34,7 @@ Slice Objects
*length* as errors.
Returns ``0`` on success and ``-1`` on error with no exception set (unless one of
- the indices was not :const:`None` and failed to be converted to an integer,
+ the indices was not ``None`` and failed to be converted to an integer,
in which case ``-1`` is returned with an exception set).
You probably do not want to use this function.
@@ -113,11 +113,13 @@ Slice Objects
Ellipsis Object
----------------
+^^^^^^^^^^^^^^^
.. c:var:: PyObject *Py_Ellipsis
- The Python ``Ellipsis`` object. This object has no methods. It needs to be
- treated just like any other object with respect to reference counts. Like
- :c:data:`Py_None` it is a singleton object.
+ The Python ``Ellipsis`` object. This object has no methods. Like
+ :c:data:`Py_None`, it is an :term:`immortal` singleton object.
+
+ .. versionchanged:: 3.12
+ :c:data:`Py_Ellipsis` is immortal.
diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst
index 4ae20e93e36785e..63a100a6f26f24c 100644
--- a/Doc/c-api/stable.rst
+++ b/Doc/c-api/stable.rst
@@ -6,9 +6,9 @@
C API Stability
***************
-Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`.
-While the C API will change with every minor release (e.g. from 3.9 to 3.10),
-most changes will be source-compatible, typically by only adding new API.
+Unless documented otherwise, Python's C API is covered by the Backwards
+Compatibility Policy, :pep:`387`.
+Most changes to it are source-compatible (typically by only adding new API).
Changing existing API or removing API is only done after a deprecation period
or to fix serious issues.
@@ -18,33 +18,63 @@ way; see :ref:`stable-abi-platform` below).
So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa,
but will need to be compiled separately for 3.9.x and 3.10.x.
+There are two tiers of C API with different stability expectations:
+
+- :ref:`Unstable API `, may change in minor versions without
+ a deprecation period. It is marked by the ``PyUnstable`` prefix in names.
+- :ref:`Limited API `, is compatible across several minor releases.
+ When :c:macro:`Py_LIMITED_API` is defined, only this subset is exposed
+ from ``Python.h``.
+
+These are discussed in more detail below.
+
Names prefixed by an underscore, such as ``_Py_InternalState``,
are private API that can change without notice even in patch releases.
+If you need to use this API, consider reaching out to
+`CPython developers `_
+to discuss adding public API for your use case.
+
+.. _unstable-c-api:
+
+Unstable C API
+==============
+
+.. index:: single: PyUnstable
+
+Any API named with the ``PyUnstable`` prefix exposes CPython implementation
+details, and may change in every minor release (e.g. from 3.9 to 3.10) without
+any deprecation warnings.
+However, it will not change in a bugfix release (e.g. from 3.10.0 to 3.10.1).
+
+It is generally intended for specialized, low-level tools like debuggers.
+
+Projects that use this API are expected to follow
+CPython development and spend extra effort adjusting to changes.
Stable Application Binary Interface
===================================
+For simplicity, this document talks about *extensions*, but the Limited API
+and Stable ABI work the same way for all uses of the API – for example,
+embedding Python.
+
+.. _limited-c-api:
+
+Limited C API
+-------------
+
Python 3.2 introduced the *Limited API*, a subset of Python's C API.
Extensions that only use the Limited API can be
compiled once and work with multiple versions of Python.
-Contents of the Limited API are :ref:`listed below `.
-
-To enable this, Python provides a *Stable ABI*: a set of symbols that will
-remain compatible across Python 3.x versions. The Stable ABI contains symbols
-exposed in the Limited API, but also other ones – for example, functions
-necessary to support older versions of the Limited API.
-
-(For simplicity, this document talks about *extensions*, but the Limited API
-and Stable ABI work the same way for all uses of the API – for example,
-embedding Python.)
+Contents of the Limited API are :ref:`listed below `.
.. c:macro:: Py_LIMITED_API
Define this macro before including ``Python.h`` to opt in to only use
the Limited API, and to select the Limited API version.
- Define ``Py_LIMITED_API`` to the value of :c:data:`PY_VERSION_HEX`
+ Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX`
corresponding to the lowest Python version your extension supports.
The extension will work without recompilation with all Python 3 releases
from the specified one onward, and can use Limited API introduced up to that
@@ -57,6 +87,19 @@ embedding Python.)
You can also define ``Py_LIMITED_API`` to ``3``. This works the same as
``0x03020000`` (Python 3.2, the version that introduced Limited API).
+
+.. _stable-abi:
+
+Stable ABI
+----------
+
+To enable this, Python provides a *Stable ABI*: a set of symbols that will
+remain compatible across Python 3.x versions.
+
+The Stable ABI contains symbols exposed in the :ref:`Limited API
+`, but also other ones – for example, functions necessary to
+support older versions of the Limited API.
+
On Windows, extensions that use the Stable ABI should be linked against
``python3.dll`` rather than a version-specific library such as
``python39.dll``.
@@ -101,9 +144,9 @@ Limited API Caveats
-------------------
Note that compiling with ``Py_LIMITED_API`` is *not* a complete guarantee that
-code conforms to the Limited API or the Stable ABI. ``Py_LIMITED_API`` only
-covers definitions, but an API also includes other issues, such as expected
-semantics.
+code conforms to the :ref:`Limited API ` or the :ref:`Stable ABI
+`. ``Py_LIMITED_API`` only covers definitions, but an API also
+includes other issues, such as expected semantics.
One issue that ``Py_LIMITED_API`` does not guard against is calling a function
with arguments that are invalid in a lower Python version.
@@ -136,9 +179,9 @@ Platform Considerations
=======================
ABI stability depends not only on Python, but also on the compiler used,
-lower-level libraries and compiler options. For the purposes of the Stable ABI,
-these details define a “platformâ€. They usually depend on the OS
-type and processor architecture
+lower-level libraries and compiler options. For the purposes of
+the :ref:`Stable ABI `, these details define a “platformâ€. They
+usually depend on the OS type and processor architecture
It is the responsibility of each particular distributor of Python
to ensure that all Python versions on a particular platform are built
@@ -147,12 +190,12 @@ This is the case with Windows and macOS releases from ``python.org`` and many
third-party distributors.
-.. _stable-abi-list:
+.. _limited-api-list:
Contents of Limited API
=======================
-Currently, the Limited API includes the following items:
+Currently, the :ref:`Limited API ` includes the following items:
.. limited-api-list::
diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst
index 183ac144c50dd3a..25cb4ed40f63e72 100644
--- a/Doc/c-api/structures.rst
+++ b/Doc/c-api/structures.rst
@@ -35,7 +35,7 @@ under :ref:`reference counting `.
.. c:type:: PyVarObject
- This is an extension of :c:type:`PyObject` that adds the :attr:`ob_size`
+ This is an extension of :c:type:`PyObject` that adds the :c:member:`~PyVarObject.ob_size`
field. This is only used for objects that have some notion of *length*.
This type does not often appear in the Python/C API.
Access to the members must be done by using the macros
@@ -152,7 +152,7 @@ under :ref:`reference counting `.
.. c:macro:: PyVarObject_HEAD_INIT(type, size)
This is a macro which expands to initialization values for a new
- :c:type:`PyVarObject` type, including the :attr:`ob_size` field.
+ :c:type:`PyVarObject` type, including the :c:member:`~PyVarObject.ob_size` field.
This macro expands to::
_PyObject_EXTRA_INIT
@@ -179,7 +179,7 @@ Implementing functions and methods
.. c:type:: PyCFunctionWithKeywords
Type of the functions used to implement Python callables in C
- with signature :const:`METH_VARARGS | METH_KEYWORDS`.
+ with signature :ref:`METH_VARARGS | METH_KEYWORDS `.
The function signature is::
PyObject *PyCFunctionWithKeywords(PyObject *self,
@@ -190,7 +190,7 @@ Implementing functions and methods
.. c:type:: _PyCFunctionFast
Type of the functions used to implement Python callables in C
- with signature :const:`METH_FASTCALL`.
+ with signature :c:macro:`METH_FASTCALL`.
The function signature is::
PyObject *_PyCFunctionFast(PyObject *self,
@@ -200,7 +200,7 @@ Implementing functions and methods
.. c:type:: _PyCFunctionFastWithKeywords
Type of the functions used to implement Python callables in C
- with signature :const:`METH_FASTCALL | METH_KEYWORDS`.
+ with signature :ref:`METH_FASTCALL | METH_KEYWORDS `.
The function signature is::
PyObject *_PyCFunctionFastWithKeywords(PyObject *self,
@@ -211,7 +211,7 @@ Implementing functions and methods
.. c:type:: PyCMethod
Type of the functions used to implement Python callables in C
- with signature :const:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS`.
+ with signature :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `.
The function signature is::
PyObject *PyCMethod(PyObject *self,
@@ -228,35 +228,38 @@ Implementing functions and methods
Structure used to describe a method of an extension type. This structure has
four fields:
- +------------------+---------------+-------------------------------+
- | Field | C Type | Meaning |
- +==================+===============+===============================+
- | :attr:`ml_name` | const char \* | name of the method |
- +------------------+---------------+-------------------------------+
- | :attr:`ml_meth` | PyCFunction | pointer to the C |
- | | | implementation |
- +------------------+---------------+-------------------------------+
- | :attr:`ml_flags` | int | flag bits indicating how the |
- | | | call should be constructed |
- +------------------+---------------+-------------------------------+
- | :attr:`ml_doc` | const char \* | points to the contents of the |
- | | | docstring |
- +------------------+---------------+-------------------------------+
-
-The :attr:`ml_meth` is a C function pointer. The functions may be of different
+ .. c:member:: const char *ml_name
+
+ Name of the method.
+
+ .. c:member:: PyCFunction ml_meth
+
+ Pointer to the C implementation.
+
+ .. c:member:: int ml_flags
+
+ Flags bits indicating how the call should be constructed.
+
+ .. c:member:: const char *ml_doc
+
+ Points to the contents of the docstring.
+
+The :c:member:`~PyMethodDef.ml_meth` is a C function pointer.
+The functions may be of different
types, but they always return :c:expr:`PyObject*`. If the function is not of
the :c:type:`PyCFunction`, the compiler will require a cast in the method table.
Even though :c:type:`PyCFunction` defines the first parameter as
:c:expr:`PyObject*`, it is common that the method implementation uses the
specific C type of the *self* object.
-The :attr:`ml_flags` field is a bitfield which can include the following flags.
+The :c:member:`~PyMethodDef.ml_flags` field is a bitfield which can include
+the following flags.
The individual flags indicate either a calling convention or a binding
convention.
There are these calling conventions:
-.. data:: METH_VARARGS
+.. c:macro:: METH_VARARGS
This is the typical calling convention, where the methods have the type
:c:type:`PyCFunction`. The function expects two :c:expr:`PyObject*` values.
@@ -266,8 +269,17 @@ There are these calling conventions:
using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`.
-.. data:: METH_VARARGS | METH_KEYWORDS
+.. c:macro:: METH_KEYWORDS
+
+ Can only be used in certain combinations with other flags:
+ :ref:`METH_VARARGS | METH_KEYWORDS `,
+ :ref:`METH_FASTCALL | METH_KEYWORDS ` and
+ :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `.
+
+.. _METH_VARARGS-METH_KEYWORDS:
+
+:c:expr:`METH_VARARGS | METH_KEYWORDS`
Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`.
The function expects three parameters: *self*, *args*, *kwargs* where
*kwargs* is a dictionary of all the keyword arguments or possibly ``NULL``
@@ -275,7 +287,7 @@ There are these calling conventions:
using :c:func:`PyArg_ParseTupleAndKeywords`.
-.. data:: METH_FASTCALL
+.. c:macro:: METH_FASTCALL
Fast calling convention supporting only positional arguments.
The methods have the type :c:type:`_PyCFunctionFast`.
@@ -287,12 +299,13 @@ There are these calling conventions:
.. versionchanged:: 3.10
- ``METH_FASTCALL`` is now part of the stable ABI.
+ ``METH_FASTCALL`` is now part of the :ref:`stable ABI `.
-.. data:: METH_FASTCALL | METH_KEYWORDS
+.. _METH_FASTCALL-METH_KEYWORDS:
- Extension of :const:`METH_FASTCALL` supporting also keyword arguments,
+:c:expr:`METH_FASTCALL | METH_KEYWORDS`
+ Extension of :c:macro:`METH_FASTCALL` supporting also keyword arguments,
with methods of type :c:type:`_PyCFunctionFastWithKeywords`.
Keyword arguments are passed the same way as in the
:ref:`vectorcall protocol `:
@@ -305,10 +318,18 @@ There are these calling conventions:
.. versionadded:: 3.7
-.. data:: METH_METHOD | METH_FASTCALL | METH_KEYWORDS
+.. c:macro:: METH_METHOD
+
+ Can only be used in the combination with other flags:
+ :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `.
+
+
+.. _METH_METHOD-METH_FASTCALL-METH_KEYWORDS:
- Extension of :const:`METH_FASTCALL | METH_KEYWORDS` supporting the *defining
- class*, that is, the class that contains the method in question.
+:c:expr:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS`
+ Extension of :ref:`METH_FASTCALL | METH_KEYWORDS `
+ supporting the *defining class*, that is,
+ the class that contains the method in question.
The defining class might be a superclass of ``Py_TYPE(self)``.
The method needs to be of type :c:type:`PyCMethod`, the same as for
@@ -318,10 +339,10 @@ There are these calling conventions:
.. versionadded:: 3.9
-.. data:: METH_NOARGS
+.. c:macro:: METH_NOARGS
Methods without parameters don't need to check whether arguments are given if
- they are listed with the :const:`METH_NOARGS` flag. They need to be of type
+ they are listed with the :c:macro:`METH_NOARGS` flag. They need to be of type
:c:type:`PyCFunction`. The first parameter is typically named *self* and will
hold a reference to the module or object instance. In all cases the second
parameter will be ``NULL``.
@@ -330,9 +351,9 @@ There are these calling conventions:
:c:macro:`Py_UNUSED` can be used to prevent a compiler warning.
-.. data:: METH_O
+.. c:macro:: METH_O
- Methods with a single object argument can be listed with the :const:`METH_O`
+ Methods with a single object argument can be listed with the :c:macro:`METH_O`
flag, instead of invoking :c:func:`PyArg_ParseTuple` with a ``"O"`` argument.
They have the type :c:type:`PyCFunction`, with the *self* parameter, and a
:c:expr:`PyObject*` parameter representing the single argument.
@@ -344,9 +365,9 @@ defined for modules. At most one of these flags may be set for any given
method.
-.. data:: METH_CLASS
+.. c:macro:: METH_CLASS
- .. index:: builtin: classmethod
+ .. index:: pair: built-in function; classmethod
The method will be passed the type object as the first parameter rather
than an instance of the type. This is used to create *class methods*,
@@ -354,9 +375,9 @@ method.
function.
-.. data:: METH_STATIC
+.. c:macro:: METH_STATIC
- .. index:: builtin: staticmethod
+ .. index:: pair: built-in function; staticmethod
The method will be passed ``NULL`` as the first parameter rather than an
instance of the type. This is used to create *static methods*, similar to
@@ -366,13 +387,13 @@ One other constant controls whether a method is loaded in place of another
definition with the same method name.
-.. data:: METH_COEXIST
+.. c:macro:: METH_COEXIST
The method will be loaded in place of existing definitions. Without
*METH_COEXIST*, the default is to skip repeated definitions. Since slot
wrappers are loaded before the method table, the existence of a
*sq_contains* slot, for example, would generate a wrapped method named
- :meth:`__contains__` and preclude the loading of a corresponding
+ :meth:`~object.__contains__` and preclude the loading of a corresponding
PyCFunction with the same name. With the flag defined, the PyCFunction
will be loaded in place of the wrapper object and will co-exist with the
slot. This is helpful because calls to PyCFunctions are optimized more
@@ -385,86 +406,71 @@ Accessing attributes of extension types
.. c:type:: PyMemberDef
Structure which describes an attribute of a type which corresponds to a C
- struct member. Its fields are:
+ struct member.
+ When defining a class, put a NULL-terminated array of these
+ structures in the :c:member:`~PyTypeObject.tp_members` slot.
- .. c:member:: const char* PyMemberDef.name
+ Its fields are, in order:
- Name of the member
+ .. c:member:: const char* name
- .. c:member:: int PyMemberDef.type
+ Name of the member.
+ A NULL value marks the end of a ``PyMemberDef[]`` array.
- The type of the member in the C struct.
+ The string should be static, no copy is made of it.
- .. c:member:: Py_ssize_t PyMemberDef.offset
+ .. c:member:: Py_ssize_t offset
The offset in bytes that the member is located on the type’s object struct.
- .. c:member:: int PyMemberDef.flags
+ .. c:member:: int type
+
+ The type of the member in the C struct.
+ See :ref:`PyMemberDef-types` for the possible values.
- Flag bits indicating if the field should be read-only or writable.
+ .. c:member:: int flags
- .. c:member:: const char* PyMemberDef.doc
+ Zero or more of the :ref:`PyMemberDef-flags`, combined using bitwise OR.
- Points to the contents of the docstring.
+ .. c:member:: const char* doc
- :c:member:`PyMemberDef.type` can be one of many ``T_`` macros corresponding to various C
- types. When the member is accessed in Python, it will be converted to the
- equivalent Python type.
-
- =============== ==================
- Macro name C type
- =============== ==================
- T_SHORT short
- T_INT int
- T_LONG long
- T_FLOAT float
- T_DOUBLE double
- T_STRING const char \*
- T_OBJECT PyObject \*
- T_OBJECT_EX PyObject \*
- T_CHAR char
- T_BYTE char
- T_UBYTE unsigned char
- T_UINT unsigned int
- T_USHORT unsigned short
- T_ULONG unsigned long
- T_BOOL char
- T_LONGLONG long long
- T_ULONGLONG unsigned long long
- T_PYSSIZET Py_ssize_t
- =============== ==================
-
- :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` differ in that
- :c:macro:`T_OBJECT` returns ``None`` if the member is ``NULL`` and
- :c:macro:`T_OBJECT_EX` raises an :exc:`AttributeError`. Try to use
- :c:macro:`T_OBJECT_EX` over :c:macro:`T_OBJECT` because :c:macro:`T_OBJECT_EX`
- handles use of the :keyword:`del` statement on that attribute more correctly
- than :c:macro:`T_OBJECT`.
-
- :c:member:`PyMemberDef.flags` can be ``0`` for write and read access or :c:macro:`READONLY` for
- read-only access. Using :c:macro:`T_STRING` for :attr:`type` implies
- :c:macro:`READONLY`. :c:macro:`T_STRING` data is interpreted as UTF-8.
- Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX`
- members can be deleted. (They are set to ``NULL``).
+ The docstring, or NULL.
+ The string should be static, no copy is made of it.
+ Typically, it is defined using :c:macro:`PyDoc_STR`.
+
+ By default (when :c:member:`~PyMemberDef.flags` is ``0``), members allow
+ both read and write access.
+ Use the :c:macro:`Py_READONLY` flag for read-only access.
+ Certain types, like :c:macro:`Py_T_STRING`, imply :c:macro:`Py_READONLY`.
+ Only :c:macro:`Py_T_OBJECT_EX` (and legacy :c:macro:`T_OBJECT`) members can
+ be deleted.
.. _pymemberdef-offsets:
- Heap allocated types (created using :c:func:`PyType_FromSpec` or similar),
- ``PyMemberDef`` may contain definitions for the special member
- ``__vectorcalloffset__``, corresponding to
+ For heap-allocated types (created using :c:func:`PyType_FromSpec` or similar),
+ ``PyMemberDef`` may contain a definition for the special member
+ ``"__vectorcalloffset__"``, corresponding to
:c:member:`~PyTypeObject.tp_vectorcall_offset` in type objects.
- These must be defined with ``T_PYSSIZET`` and ``READONLY``, for example::
+ These must be defined with ``Py_T_PYSSIZET`` and ``Py_READONLY``, for example::
static PyMemberDef spam_type_members[] = {
- {"__vectorcalloffset__", T_PYSSIZET, offsetof(Spam_object, vectorcall), READONLY},
+ {"__vectorcalloffset__", Py_T_PYSSIZET,
+ offsetof(Spam_object, vectorcall), Py_READONLY},
{NULL} /* Sentinel */
};
+ (You may need to ``#include `` for :c:func:`!offsetof`.)
+
The legacy offsets :c:member:`~PyTypeObject.tp_dictoffset` and
- :c:member:`~PyTypeObject.tp_weaklistoffset` are still supported, but extensions are
- strongly encouraged to use ``Py_TPFLAGS_MANAGED_DICT`` and
- ``Py_TPFLAGS_MANAGED_WEAKREF`` instead.
+ :c:member:`~PyTypeObject.tp_weaklistoffset` can be defined similarly using
+ ``"__dictoffset__"`` and ``"__weaklistoffset__"`` members, but extensions
+ are strongly encouraged to use :c:macro:`Py_TPFLAGS_MANAGED_DICT` and
+ :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead.
+
+ .. versionchanged:: 3.12
+ ``PyMemberDef`` is always available.
+ Previously, it required including ``"structmember.h"``.
.. c:function:: PyObject* PyMember_GetOne(const char *obj_addr, struct PyMemberDef *m)
@@ -472,6 +478,10 @@ Accessing attributes of extension types
attribute is described by ``PyMemberDef`` *m*. Returns ``NULL``
on error.
+ .. versionchanged:: 3.12
+
+ ``PyMember_GetOne`` is always available.
+ Previously, it required including ``"structmember.h"``.
.. c:function:: int PyMember_SetOne(char *obj_addr, struct PyMemberDef *m, PyObject *o)
@@ -479,29 +489,185 @@ Accessing attributes of extension types
The attribute to set is described by ``PyMemberDef`` *m*. Returns ``0``
if successful and a negative value on failure.
+ .. versionchanged:: 3.12
+
+ ``PyMember_SetOne`` is always available.
+ Previously, it required including ``"structmember.h"``.
+
+.. _PyMemberDef-flags:
+
+Member flags
+^^^^^^^^^^^^
+
+The following flags can be used with :c:member:`PyMemberDef.flags`:
+
+.. c:macro:: Py_READONLY
+
+ Not writable.
+
+.. c:macro:: Py_AUDIT_READ
+
+ Emit an ``object.__getattr__`` :ref:`audit event `
+ before reading.
+
+.. c:macro:: Py_RELATIVE_OFFSET
+
+ Indicates that the :c:member:`~PyMemberDef.offset` of this ``PyMemberDef``
+ entry indicates an offset from the subclass-specific data, rather than
+ from ``PyObject``.
+
+ Can only be used as part of :c:member:`Py_tp_members `
+ :c:type:`slot ` when creating a class using negative
+ :c:member:`~PyType_Spec.basicsize`.
+ It is mandatory in that case.
+
+ This flag is only used in :c:type:`PyTypeSlot`.
+ When setting :c:member:`~PyTypeObject.tp_members` during
+ class creation, Python clears it and sets
+ :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct.
+
+.. index::
+ single: READ_RESTRICTED
+ single: WRITE_RESTRICTED
+ single: RESTRICTED
+
+.. versionchanged:: 3.10
+
+ The :c:macro:`!RESTRICTED`, :c:macro:`!READ_RESTRICTED` and
+ :c:macro:`!WRITE_RESTRICTED` macros available with
+ ``#include "structmember.h"`` are deprecated.
+ :c:macro:`!READ_RESTRICTED` and :c:macro:`!RESTRICTED` are equivalent to
+ :c:macro:`Py_AUDIT_READ`; :c:macro:`!WRITE_RESTRICTED` does nothing.
+
+.. index::
+ single: READONLY
+
+.. versionchanged:: 3.12
+
+ The :c:macro:`!READONLY` macro was renamed to :c:macro:`Py_READONLY`.
+ The :c:macro:`!PY_AUDIT_READ` macro was renamed with the ``Py_`` prefix.
+ The new names are now always available.
+ Previously, these required ``#include "structmember.h"``.
+ The header is still available and it provides the old names.
+
+.. _PyMemberDef-types:
+
+Member types
+^^^^^^^^^^^^
+
+:c:member:`PyMemberDef.type` can be one of the following macros corresponding
+to various C types.
+When the member is accessed in Python, it will be converted to the
+equivalent Python type.
+When it is set from Python, it will be converted back to the C type.
+If that is not possible, an exception such as :exc:`TypeError` or
+:exc:`ValueError` is raised.
+
+Unless marked (D), attributes defined this way cannot be deleted
+using e.g. :keyword:`del` or :py:func:`delattr`.
+
+================================ ============================= ======================
+Macro name C type Python type
+================================ ============================= ======================
+.. c:macro:: Py_T_BYTE :c:expr:`char` :py:class:`int`
+.. c:macro:: Py_T_SHORT :c:expr:`short` :py:class:`int`
+.. c:macro:: Py_T_INT :c:expr:`int` :py:class:`int`
+.. c:macro:: Py_T_LONG :c:expr:`long` :py:class:`int`
+.. c:macro:: Py_T_LONGLONG :c:expr:`long long` :py:class:`int`
+.. c:macro:: Py_T_UBYTE :c:expr:`unsigned char` :py:class:`int`
+.. c:macro:: Py_T_UINT :c:expr:`unsigned int` :py:class:`int`
+.. c:macro:: Py_T_USHORT :c:expr:`unsigned short` :py:class:`int`
+.. c:macro:: Py_T_ULONG :c:expr:`unsigned long` :py:class:`int`
+.. c:macro:: Py_T_ULONGLONG :c:expr:`unsigned long long` :py:class:`int`
+.. c:macro:: Py_T_PYSSIZET :c:expr:`Py_ssize_t` :py:class:`int`
+.. c:macro:: Py_T_FLOAT :c:expr:`float` :py:class:`float`
+.. c:macro:: Py_T_DOUBLE :c:expr:`double` :py:class:`float`
+.. c:macro:: Py_T_BOOL :c:expr:`char` :py:class:`bool`
+ (written as 0 or 1)
+.. c:macro:: Py_T_STRING :c:expr:`const char *` (*) :py:class:`str` (RO)
+.. c:macro:: Py_T_STRING_INPLACE :c:expr:`const char[]` (*) :py:class:`str` (RO)
+.. c:macro:: Py_T_CHAR :c:expr:`char` (0-127) :py:class:`str` (**)
+.. c:macro:: Py_T_OBJECT_EX :c:expr:`PyObject *` :py:class:`object` (D)
+================================ ============================= ======================
+
+ (*): Zero-terminated, UTF8-encoded C string.
+ With :c:macro:`!Py_T_STRING` the C representation is a pointer;
+ with :c:macro:`!Py_T_STRING_INLINE` the string is stored directly
+ in the structure.
+
+ (**): String of length 1. Only ASCII is accepted.
+
+ (RO): Implies :c:macro:`Py_READONLY`.
+
+ (D): Can be deleted, in which case the pointer is set to ``NULL``.
+ Reading a ``NULL`` pointer raises :py:exc:`AttributeError`.
+
+.. index::
+ single: T_BYTE
+ single: T_SHORT
+ single: T_INT
+ single: T_LONG
+ single: T_LONGLONG
+ single: T_UBYTE
+ single: T_USHORT
+ single: T_UINT
+ single: T_ULONG
+ single: T_ULONGULONG
+ single: T_PYSSIZET
+ single: T_FLOAT
+ single: T_DOUBLE
+ single: T_BOOL
+ single: T_CHAR
+ single: T_STRING
+ single: T_STRING_INPLACE
+ single: T_OBJECT_EX
+ single: structmember.h
+
+.. versionadded:: 3.12
+
+ In previous versions, the macros were only available with
+ ``#include "structmember.h"`` and were named without the ``Py_`` prefix
+ (e.g. as ``T_INT``).
+ The header is still available and contains the old names, along with
+ the following deprecated types:
+
+ .. c:macro:: T_OBJECT
+
+ Like ``Py_T_OBJECT_EX``, but ``NULL`` is converted to ``None``.
+ This results in surprising behavior in Python: deleting the attribute
+ effectively sets it to ``None``.
+
+ .. c:macro:: T_NONE
+
+ Always ``None``. Must be used with :c:macro:`Py_READONLY`.
+
+Defining Getters and Setters
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. c:type:: PyGetSetDef
Structure to define property-like access for a type. See also description of
the :c:member:`PyTypeObject.tp_getset` slot.
- +-------------+------------------+-----------------------------------+
- | Field | C Type | Meaning |
- +=============+==================+===================================+
- | name | const char \* | attribute name |
- +-------------+------------------+-----------------------------------+
- | get | getter | C function to get the attribute |
- +-------------+------------------+-----------------------------------+
- | set | setter | optional C function to set or |
- | | | delete the attribute, if omitted |
- | | | the attribute is readonly |
- +-------------+------------------+-----------------------------------+
- | doc | const char \* | optional docstring |
- +-------------+------------------+-----------------------------------+
- | closure | void \* | optional function pointer, |
- | | | providing additional data for |
- | | | getter and setter |
- +-------------+------------------+-----------------------------------+
+ .. c:member:: const char* name
+
+ attribute name
+
+ .. c:member:: getter get
+
+ C function to get the attribute.
+
+ .. c:member:: setter set
+
+ Optional C function to set or delete the attribute, if omitted the attribute is readonly.
+
+ .. c:member:: const char* doc
+
+ optional docstring
+
+ .. c:member:: void* closure
+
+ Optional function pointer, providing additional data for getter and setter.
The ``get`` function takes one :c:expr:`PyObject*` parameter (the
instance) and a function pointer (the associated ``closure``)::
diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst
index 6fc8a3aff956862..e3c54b075114ff7 100644
--- a/Doc/c-api/sys.rst
+++ b/Doc/c-api/sys.rst
@@ -8,8 +8,9 @@ Operating System Utilities
.. c:function:: PyObject* PyOS_FSPath(PyObject *path)
Return the file system representation for *path*. If the object is a
- :class:`str` or :class:`bytes` object, then its reference count is
- incremented. If the object implements the :class:`os.PathLike` interface,
+ :class:`str` or :class:`bytes` object, then a new
+ :term:`strong reference` is returned.
+ If the object implements the :class:`os.PathLike` interface,
then :meth:`~os.PathLike.__fspath__` is returned as long as it is a
:class:`str` or :class:`bytes` object. Otherwise :exc:`TypeError` is raised
and ``NULL`` is returned.
@@ -97,16 +98,16 @@ Operating System Utilities
.. c:function:: int PyOS_CheckStack()
Return true when the interpreter runs out of stack space. This is a reliable
- check, but is only available when :const:`USE_STACKCHECK` is defined (currently
+ check, but is only available when :c:macro:`USE_STACKCHECK` is defined (currently
on certain versions of Windows using the Microsoft Visual C++ compiler).
- :const:`USE_STACKCHECK` will be defined automatically; you should never
+ :c:macro:`USE_STACKCHECK` will be defined automatically; you should never
change the definition in your own code.
.. c:function:: PyOS_sighandler_t PyOS_getsig(int i)
Return the current signal handler for signal *i*. This is a thin wrapper around
- either :c:func:`sigaction` or :c:func:`signal`. Do not call those functions
+ either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions
directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void
(\*)(int)`.
@@ -114,7 +115,7 @@ Operating System Utilities
.. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h)
Set the signal handler for signal *i* to be *h*; return the old signal handler.
- This is a thin wrapper around either :c:func:`sigaction` or :c:func:`signal`. Do
+ This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do
not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef
alias for :c:expr:`void (\*)(int)`.
@@ -167,7 +168,7 @@ Operating System Utilities
.. versionchanged:: 3.8
The function now uses the UTF-8 encoding on Windows if
- :c:member:`PyConfig.legacy_windows_fs_encoding` is zero;
+ :c:member:`PyPreConfig.legacy_windows_fs_encoding` is zero;
.. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos)
@@ -209,7 +210,7 @@ Operating System Utilities
.. versionchanged:: 3.8
The function now uses the UTF-8 encoding on Windows if
- :c:member:`PyConfig.legacy_windows_fs_encoding` is zero.
+ :c:member:`PyPreConfig.legacy_windows_fs_encoding` is zero.
.. _systemfunctions:
@@ -237,45 +238,8 @@ accessible to C code. They all work with the current interpreter thread's
Reset :data:`sys.warnoptions` to an empty list. This function may be
called prior to :c:func:`Py_Initialize`.
-.. c:function:: void PySys_AddWarnOption(const wchar_t *s)
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.warnoptions` should be used instead, see :ref:`Python
- Initialization Configuration `.
-
- Append *s* to :data:`sys.warnoptions`. This function must be called prior
- to :c:func:`Py_Initialize` in order to affect the warnings filter list.
-
- .. deprecated:: 3.11
-
-.. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode)
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.warnoptions` should be used instead, see :ref:`Python
- Initialization Configuration `.
-
- Append *unicode* to :data:`sys.warnoptions`.
-
- Note: this function is not currently usable from outside the CPython
- implementation, as it must be called prior to the implicit import of
- :mod:`warnings` in :c:func:`Py_Initialize` to be effective, but can't be
- called until enough of the runtime has been initialized to permit the
- creation of Unicode objects.
-
- .. deprecated:: 3.11
-
-.. c:function:: void PySys_SetPath(const wchar_t *path)
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.module_search_paths` and
- :c:member:`PyConfig.module_search_paths_set` should be used instead, see
- :ref:`Python Initialization Configuration `.
-
- Set :data:`sys.path` to a list object of paths found in *path* which should
- be a list of paths separated with the platform's search path delimiter
- (``:`` on Unix, ``;`` on Windows).
-
- .. deprecated:: 3.11
+ .. deprecated-removed:: 3.13 3.15
+ Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead.
.. c:function:: void PySys_WriteStdout(const char *format, ...)
@@ -313,20 +277,6 @@ accessible to C code. They all work with the current interpreter thread's
.. versionadded:: 3.2
-.. c:function:: void PySys_AddXOption(const wchar_t *s)
-
- This API is kept for backward compatibility: setting
- :c:member:`PyConfig.xoptions` should be used instead, see :ref:`Python
- Initialization Configuration `.
-
- Parse *s* as a set of :option:`-X` options and add them to the current
- options mapping as returned by :c:func:`PySys_GetXOptions`. This function
- may be called prior to :c:func:`Py_Initialize`.
-
- .. versionadded:: 3.2
-
- .. deprecated:: 3.11
-
.. c:function:: PyObject *PySys_GetXOptions()
Return the current dictionary of :option:`-X` options, similarly to
@@ -341,19 +291,24 @@ accessible to C code. They all work with the current interpreter thread's
Raise an auditing event with any active hooks. Return zero for success
and non-zero with an exception set on failure.
+ The *event* string argument must not be *NULL*.
+
If any hooks have been added, *format* and other arguments will be used
to construct a tuple to pass. Apart from ``N``, the same format characters
as used in :c:func:`Py_BuildValue` are available. If the built value is not
- a tuple, it will be added into a single-element tuple. (The ``N`` format
- option consumes a reference, but since there is no way to know whether
- arguments to this function will be consumed, using it may cause reference
- leaks.)
+ a tuple, it will be added into a single-element tuple.
+
+ The ``N`` format option must not be used. It consumes a reference, but since
+ there is no way to know whether arguments to this function will be consumed,
+ using it may cause reference leaks.
Note that ``#`` format characters should always be treated as
:c:type:`Py_ssize_t`, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined.
:func:`sys.audit` performs the same function from Python code.
+ See also :c:func:`PySys_AuditTuple`.
+
.. versionadded:: 3.8
.. versionchanged:: 3.8.2
@@ -362,6 +317,14 @@ accessible to C code. They all work with the current interpreter thread's
unavoidable deprecation warning was raised.
+.. c:function:: int PySys_AuditTuple(const char *event, PyObject *args)
+
+ Similar to :c:func:`PySys_Audit`, but pass arguments as a Python object.
+ *args* must be a :class:`tuple`. To pass no arguments, *args* can be *NULL*.
+
+ .. versionadded:: 3.13
+
+
.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData)
Append the callable *hook* to the list of active auditing hooks.
@@ -414,7 +377,7 @@ Process Control
This function should only be invoked when a condition is detected that would
make it dangerous to continue using the Python interpreter; e.g., when the
object administration appears to be corrupted. On Unix, the standard C library
- function :c:func:`abort` is called which will attempt to produce a :file:`core`
+ function :c:func:`!abort` is called which will attempt to produce a :file:`core`
file.
The ``Py_FatalError()`` function is replaced with a macro which logs
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
index 5acddf7849aa33f..b3710560ebe7ac9 100644
--- a/Doc/c-api/tuple.rst
+++ b/Doc/c-api/tuple.rst
@@ -5,7 +5,7 @@
Tuple Objects
-------------
-.. index:: object: tuple
+.. index:: pair: object; tuple
.. c:type:: PyTupleObject
@@ -89,6 +89,9 @@ Tuple Objects
Like :c:func:`PyTuple_SetItem`, but does no error checking, and should *only* be
used to fill in brand new tuples.
+ Bounds checking is performed as an assertion if Python is built in
+ :ref:`debug mode ` or :option:`with assertions <--with-assertions>`.
+
.. note::
This function "steals" a reference to *o*, and, unlike
@@ -111,6 +114,8 @@ Tuple Objects
raises :exc:`MemoryError` or :exc:`SystemError`.
+.. _struct-sequence-objects:
+
Struct Sequence Objects
-----------------------
@@ -142,39 +147,39 @@ type.
Contains the meta information of a struct sequence type to create.
- +-------------------+------------------------------+--------------------------------------+
- | Field | C Type | Meaning |
- +===================+==============================+======================================+
- | ``name`` | ``const char *`` | name of the struct sequence type |
- +-------------------+------------------------------+--------------------------------------+
- | ``doc`` | ``const char *`` | pointer to docstring for the type |
- | | | or ``NULL`` to omit |
- +-------------------+------------------------------+--------------------------------------+
- | ``fields`` | ``PyStructSequence_Field *`` | pointer to ``NULL``-terminated array |
- | | | with field names of the new type |
- +-------------------+------------------------------+--------------------------------------+
- | ``n_in_sequence`` | ``int`` | number of fields visible to the |
- | | | Python side (if used as tuple) |
- +-------------------+------------------------------+--------------------------------------+
+ .. c:member:: const char *name
+
+ Name of the struct sequence type.
+
+ .. c:member:: const char *doc
+
+ Pointer to docstring for the type or ``NULL`` to omit.
+
+ .. c:member:: PyStructSequence_Field *fields
+
+ Pointer to ``NULL``-terminated array with field names of the new type.
+
+ .. c:member:: int n_in_sequence
+
+ Number of fields visible to the Python side (if used as tuple).
.. c:type:: PyStructSequence_Field
Describes a field of a struct sequence. As a struct sequence is modeled as a
tuple, all fields are typed as :c:expr:`PyObject*`. The index in the
- :attr:`fields` array of the :c:type:`PyStructSequence_Desc` determines which
+ :c:member:`~PyStructSequence_Desc.fields` array of
+ the :c:type:`PyStructSequence_Desc` determines which
field of the struct sequence is described.
- +-----------+------------------+-----------------------------------------+
- | Field | C Type | Meaning |
- +===========+==================+=========================================+
- | ``name`` | ``const char *`` | name for the field or ``NULL`` to end |
- | | | the list of named fields, set to |
- | | | :c:data:`PyStructSequence_UnnamedField` |
- | | | to leave unnamed |
- +-----------+------------------+-----------------------------------------+
- | ``doc`` | ``const char *`` | field docstring or ``NULL`` to omit |
- +-----------+------------------+-----------------------------------------+
+ .. c:member:: const char *name
+
+ Name for the field or ``NULL`` to end the list of named fields,
+ set to :c:data:`PyStructSequence_UnnamedField` to leave unnamed.
+
+ .. c:member:: const char *doc
+
+ Field docstring or ``NULL`` to omit.
.. c:var:: const char * const PyStructSequence_UnnamedField
@@ -194,12 +199,17 @@ type.
.. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos)
Return the object at position *pos* in the struct sequence pointed to by *p*.
- No bounds checking is performed.
+
+ Bounds checking is performed as an assertion if Python is built in
+ :ref:`debug mode ` or :option:`with assertions <--with-assertions>`.
.. c:function:: PyObject* PyStructSequence_GET_ITEM(PyObject *p, Py_ssize_t pos)
- Macro equivalent of :c:func:`PyStructSequence_GetItem`.
+ Alias to :c:func:`PyStructSequence_GetItem`.
+
+ .. versionchanged:: 3.13
+ Now implemented as an alias to :c:func:`PyStructSequence_GetItem`.
.. c:function:: void PyStructSequence_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o)
@@ -208,6 +218,9 @@ type.
:c:func:`PyTuple_SET_ITEM`, this should only be used to fill in brand new
instances.
+ Bounds checking is performed as an assertion if Python is built in
+ :ref:`debug mode ` or :option:`with assertions <--with-assertions>`.
+
.. note::
This function "steals" a reference to *o*.
@@ -215,9 +228,7 @@ type.
.. c:function:: void PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o)
- Similar to :c:func:`PyStructSequence_SetItem`, but implemented as a static
- inlined function.
-
- .. note::
+ Alias to :c:func:`PyStructSequence_SetItem`.
- This function "steals" a reference to *o*.
+ .. versionchanged:: 3.13
+ Now implemented as an alias to :c:func:`PyStructSequence_SetItem`.
diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 7b5d1fac40ed87d..5aaa8147dd3176a 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -5,7 +5,7 @@
Type Objects
------------
-.. index:: object: type
+.. index:: pair: object; type
.. c:type:: PyTypeObject
@@ -42,7 +42,7 @@ Type Objects
Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily
meant for use with ``Py_LIMITED_API``; the individual flag bits are
guaranteed to be stable across Python releases, but access to
- :c:member:`~PyTypeObject.tp_flags` itself is not part of the limited API.
+ :c:member:`~PyTypeObject.tp_flags` itself is not part of the :ref:`limited API `.
.. versionadded:: 3.2
@@ -50,6 +50,23 @@ Type Objects
The return type is now ``unsigned long`` rather than ``long``.
+.. c:function:: PyObject* PyType_GetDict(PyTypeObject* type)
+
+ Return the type object's internal namespace, which is otherwise only
+ exposed via a read-only proxy (``cls.__dict__``). This is a
+ replacement for accessing :c:member:`~PyTypeObject.tp_dict` directly.
+ The returned dictionary must be treated as read-only.
+
+ This function is meant for specific embedding and language-binding cases,
+ where direct access to the dict is necessary and indirect access
+ (e.g. via the proxy or :c:func:`PyObject_GetAttr`) isn't adequate.
+
+ Extension modules should continue to use ``tp_dict``,
+ directly or indirectly, when setting up their own types.
+
+ .. versionadded:: 3.12
+
+
.. c:function:: void PyType_Modified(PyTypeObject *type)
Invalidate the internal lookup cache for the type and all of its
@@ -86,7 +103,7 @@ Type Objects
:c:func:`PyType_AddWatcher` will be called whenever
:c:func:`PyType_Modified` reports a change to *type*. (The callback may be
called only once for a series of consecutive modifications to *type*, if
- :c:func:`PyType_Lookup` is not called on *type* between the modifications;
+ :c:func:`!_PyType_Lookup` is not called on *type* between the modifications;
this is an implementation detail and subject to change.)
An extension should never call ``PyType_Watch`` with a *watcher_id* that was
@@ -115,7 +132,7 @@ Type Objects
.. c:function:: int PyType_IS_GC(PyTypeObject *o)
Return true if the type object includes support for the cycle detector; this
- tests the type flag :const:`Py_TPFLAGS_HAVE_GC`.
+ tests the type flag :c:macro:`Py_TPFLAGS_HAVE_GC`.
.. c:function:: int PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b)
@@ -148,10 +165,10 @@ Type Objects
.. note::
If some of the base classes implements the GC protocol and the provided
- type does not include the :const:`Py_TPFLAGS_HAVE_GC` in its flags, then
+ type does not include the :c:macro:`Py_TPFLAGS_HAVE_GC` in its flags, then
the GC protocol will be automatically implemented from its parents. On
the contrary, if the type being created does include
- :const:`Py_TPFLAGS_HAVE_GC` in its flags then it **must** implement the
+ :c:macro:`Py_TPFLAGS_HAVE_GC` in its flags then it **must** implement the
GC protocol itself by at least implementing the
:c:member:`~PyTypeObject.tp_traverse` handle.
@@ -198,7 +215,7 @@ Type Objects
``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses
are not necessarily defined in the same module as their superclass.
See :c:type:`PyCMethod` to get the class that defines the method.
- See :c:func:`PyType_GetModuleByDef` for cases when ``PyCMethod`` cannot
+ See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot
be used.
.. versionadded:: 3.9
@@ -232,6 +249,15 @@ Type Objects
.. versionadded:: 3.11
+.. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type)
+
+ Attempt to assign a version tag to the given type.
+
+ Returns 1 if the type already had a valid version tag or a new one was
+ assigned, or 0 if a new tag could not be assigned.
+
+ .. versionadded:: 3.12
+
Creating Heap-Allocated Types
.............................
@@ -242,13 +268,18 @@ The following functions and structs are used to create
.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases)
Create and return a :ref:`heap type ` from the *spec*
- (see :const:`Py_TPFLAGS_HEAPTYPE`).
+ (see :c:macro:`Py_TPFLAGS_HEAPTYPE`).
The metaclass *metaclass* is used to construct the resulting type object.
When *metaclass* is ``NULL``, the metaclass is derived from *bases*
(or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below).
- Note that metaclasses that override
- :c:member:`~PyTypeObject.tp_new` are not supported.
+
+ Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not
+ supported, except if ``tp_new`` is ``NULL``.
+ (For backwards compatibility, other ``PyType_From*`` functions allow
+ such metaclasses. They ignore ``tp_new``, which may result in incomplete
+ initialization. This is deprecated and in Python 3.14+ such metaclasses will
+ not be supported.)
The *bases* argument can be used to specify base classes; it can either
be only one class or a tuple of classes.
@@ -296,6 +327,11 @@ The following functions and structs are used to create
The function now finds and uses a metaclass corresponding to the provided
base classes. Previously, only :class:`type` instances were returned.
+ The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*.
+ which may result in incomplete initialization.
+ Creating classes whose metaclass overrides
+ :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it
+ will be no longer allowed.
.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
@@ -308,6 +344,12 @@ The following functions and structs are used to create
The function now finds and uses a metaclass corresponding to the provided
base classes. Previously, only :class:`type` instances were returned.
+ The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*.
+ which may result in incomplete initialization.
+ Creating classes whose metaclass overrides
+ :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it
+ will be no longer allowed.
+
.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)
Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``.
@@ -318,41 +360,94 @@ The following functions and structs are used to create
base classes provided in *Py_tp_base[s]* slots.
Previously, only :class:`type` instances were returned.
+ The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*.
+ which may result in incomplete initialization.
+ Creating classes whose metaclass overrides
+ :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it
+ will be no longer allowed.
+
+.. raw:: html
+
+
+
+
+
+
+
+
.. c:type:: PyType_Spec
Structure defining a type's behavior.
- .. c:member:: const char* PyType_Spec.name
+ .. c:member:: const char* name
Name of the type, used to set :c:member:`PyTypeObject.tp_name`.
- .. c:member:: int PyType_Spec.basicsize
- .. c:member:: int PyType_Spec.itemsize
+ .. c:member:: int basicsize
+
+ If positive, specifies the size of the instance in bytes.
+ It is used to set :c:member:`PyTypeObject.tp_basicsize`.
+
+ If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize`
+ should be inherited.
+
+ If negative, the absolute value specifies how much space instances of the
+ class need *in addition* to the superclass.
+ Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific
+ memory reserved this way.
+
+ .. versionchanged:: 3.12
+
+ Previously, this field could not be negative.
+
+ .. c:member:: int itemsize
+
+ Size of one element of a variable-size type, in bytes.
+ Used to set :c:member:`PyTypeObject.tp_itemsize`.
+ See ``tp_itemsize`` documentation for caveats.
- Size of the instance in bytes, used to set
- :c:member:`PyTypeObject.tp_basicsize` and
- :c:member:`PyTypeObject.tp_itemsize`.
+ If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited.
+ Extending arbitrary variable-sized classes is dangerous,
+ since some types use a fixed offset for variable-sized memory,
+ which can then overlap fixed-sized memory used by a subclass.
+ To help prevent mistakes, inheriting ``itemsize`` is only possible
+ in the following situations:
- .. c:member:: int PyType_Spec.flags
+ - The base is not variable-sized (its
+ :c:member:`~PyTypeObject.tp_itemsize`).
+ - The requested :c:member:`PyType_Spec.basicsize` is positive,
+ suggesting that the memory layout of the base class is known.
+ - The requested :c:member:`PyType_Spec.basicsize` is zero,
+ suggesting that the subclass does not access the instance's memory
+ directly.
+ - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag.
+
+ .. c:member:: unsigned int flags
Type flags, used to set :c:member:`PyTypeObject.tp_flags`.
If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set,
:c:func:`PyType_FromSpecWithBases` sets it automatically.
- .. c:member:: PyType_Slot *PyType_Spec.slots
+ .. c:member:: PyType_Slot *slots
Array of :c:type:`PyType_Slot` structures.
Terminated by the special slot value ``{0, NULL}``.
Each slot ID should be specified at most once.
+.. raw:: html
+
+
+
+
+
.. c:type:: PyType_Slot
Structure defining optional functionality of a type, containing a slot ID
and a value pointer.
- .. c:member:: int PyType_Slot.slot
+ .. c:member:: int slot
A slot ID.
@@ -366,26 +461,39 @@ The following functions and structs are used to create
* ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add`
* ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length`
- The following fields cannot be set at all using :c:type:`PyType_Spec` and
- :c:type:`PyType_Slot`:
+ The following “offset†fields cannot be set using :c:type:`PyType_Slot`:
+
+ * :c:member:`~PyTypeObject.tp_weaklistoffset`
+ (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible)
+ * :c:member:`~PyTypeObject.tp_dictoffset`
+ (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible)
+ * :c:member:`~PyTypeObject.tp_vectorcall_offset`
+ (use ``"__vectorcalloffset__"`` in
+ :ref:`PyMemberDef `)
+
+ If it is not possible to switch to a ``MANAGED`` flag (for example,
+ for vectorcall or to support Python older than 3.12), specify the
+ offset in :c:member:`Py_tp_members `.
+ See :ref:`PyMemberDef documentation `
+ for details.
+
+ The following fields cannot be set at all when creating a heap type:
- * :c:member:`~PyTypeObject.tp_dict`
- * :c:member:`~PyTypeObject.tp_mro`
- * :c:member:`~PyTypeObject.tp_cache`
- * :c:member:`~PyTypeObject.tp_subclasses`
- * :c:member:`~PyTypeObject.tp_weaklist`
* :c:member:`~PyTypeObject.tp_vectorcall`
- * :c:member:`~PyTypeObject.tp_weaklistoffset`
- (use :const:`Py_TPFLAGS_MANAGED_WEAKREF` instead)
- * :c:member:`~PyTypeObject.tp_dictoffset`
- (use :const:`Py_TPFLAGS_MANAGED_DICT` instead)
- * :c:member:`~PyTypeObject.tp_vectorcall_offset`
- (see :ref:`PyMemberDef `)
+ (use :c:member:`~PyTypeObject.tp_new` and/or
+ :c:member:`~PyTypeObject.tp_init`)
+
+ * Internal fields:
+ :c:member:`~PyTypeObject.tp_dict`,
+ :c:member:`~PyTypeObject.tp_mro`,
+ :c:member:`~PyTypeObject.tp_cache`,
+ :c:member:`~PyTypeObject.tp_subclasses`, and
+ :c:member:`~PyTypeObject.tp_weaklist`.
Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be
problematic on some platforms.
To avoid issues, use the *bases* argument of
- :py:func:`PyType_FromSpecWithBases` instead.
+ :c:func:`PyType_FromSpecWithBases` instead.
.. versionchanged:: 3.9
@@ -394,9 +502,9 @@ The following functions and structs are used to create
.. versionchanged:: 3.11
:c:member:`~PyBufferProcs.bf_getbuffer` and
:c:member:`~PyBufferProcs.bf_releasebuffer` are now available
- under the limited API.
+ under the :ref:`limited API `.
- .. c:member:: void *PyType_Slot.pfunc
+ .. c:member:: void *pfunc
The desired value of the slot. In most cases, this is a pointer
to a function.
diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst
index 4c1957a2a1dbca0..98fe68737deb81c 100644
--- a/Doc/c-api/typehints.rst
+++ b/Doc/c-api/typehints.rst
@@ -35,7 +35,7 @@ two types exist -- :ref:`GenericAlias ` and
...
}
- .. seealso:: The data model method :meth:`__class_getitem__`.
+ .. seealso:: The data model method :meth:`~object.__class_getitem__`.
.. versionadded:: 3.9
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 8ccdece3efc54c9..10c05beda7c66f0 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -147,17 +147,25 @@ Quick Reference
+------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+
| :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | |
+------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+
+ | [:c:member:`~PyTypeObject.tp_watched`] | unsigned char | | | | | |
+ +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+
.. [#slots]
- A slot name in parentheses indicates it is (effectively) deprecated.
- Names in angle brackets should be treated as read-only.
- Names in square brackets are for internal use only.
- "" (as a prefix) means the field is required (must be non-``NULL``).
+
+ **()**: A slot name in parentheses indicates it is (effectively) deprecated.
+
+ **<>**: Names in angle brackets should be initially set to ``NULL`` and
+ treated as read-only.
+
+ **[]**: Names in square brackets are for internal use only.
+
+ **** (as a prefix) means the field is required (must be non-``NULL``).
+
.. [#cols] Columns:
- **"O"**: set on :c:type:`PyBaseObject_Type`
+ **"O"**: set on :c:data:`PyBaseObject_Type`
- **"T"**: set on :c:type:`PyType_Type`
+ **"T"**: set on :c:data:`PyType_Type`
**"D"**: default (if slot is set to ``NULL``)
@@ -444,6 +452,7 @@ slot typedefs
| | | |
| | :c:type:`PyObject` * | |
| | :c:type:`Py_ssize_t` | |
+| | :c:type:`PyObject` * | |
+-----------------------------+-----------------------------+----------------------+
| :c:type:`objobjproc` | .. line-block:: | int |
| | | |
@@ -476,17 +485,17 @@ PyObject Slots
--------------
The type object structure extends the :c:type:`PyVarObject` structure. The
-:attr:`ob_size` field is used for dynamic types (created by :func:`type_new`,
+:c:member:`~PyVarObject.ob_size` field is used for dynamic types (created by :c:func:`!type_new`,
usually called from a class statement). Note that :c:data:`PyType_Type` (the
metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that its instances (i.e.
-type objects) *must* have the :attr:`ob_size` field.
+type objects) *must* have the :c:member:`~PyVarObject.ob_size` field.
.. c:member:: Py_ssize_t PyObject.ob_refcnt
This is the type object's reference count, initialized to ``1`` by the
``PyObject_HEAD_INIT`` macro. Note that for :ref:`statically allocated type
- objects `, the type's instances (objects whose :attr:`ob_type`
+ objects `, the type's instances (objects whose :c:member:`~PyObject.ob_type`
points back to the type) do *not* count as references. But for
:ref:`dynamically allocated type objects `, the instances *do*
count as references.
@@ -510,8 +519,8 @@ type objects) *must* have the :attr:`ob_size` field.
Foo_Type.ob_type = &PyType_Type;
This should be done before any instances of the type are created.
- :c:func:`PyType_Ready` checks if :attr:`ob_type` is ``NULL``, and if so,
- initializes it to the :attr:`ob_type` field of the base class.
+ :c:func:`PyType_Ready` checks if :c:member:`~PyObject.ob_type` is ``NULL``, and if so,
+ initializes it to the :c:member:`~PyObject.ob_type` field of the base class.
:c:func:`PyType_Ready` will not change this field if it is non-zero.
**Inheritance:**
@@ -519,28 +528,6 @@ type objects) *must* have the :attr:`ob_size` field.
This field is inherited by subtypes.
-.. c:member:: PyObject* PyObject._ob_next
- PyObject* PyObject._ob_prev
-
- These fields are only present when the macro ``Py_TRACE_REFS`` is defined
- (see the :option:`configure --with-trace-refs option <--with-trace-refs>`).
-
- Their initialization to ``NULL`` is taken care of by the
- ``PyObject_HEAD_INIT`` macro. For :ref:`statically allocated objects
- `, these fields always remain ``NULL``. For :ref:`dynamically
- allocated objects `, these two fields are used to link the
- object into a doubly linked list of *all* live objects on the heap.
-
- This could be used for various debugging purposes; currently the only uses
- are the :func:`sys.getobjects` function and to print the objects that are
- still alive at the end of a run when the environment variable
- :envvar:`PYTHONDUMPREFS` is set.
-
- **Inheritance:**
-
- These fields are not inherited by subtypes.
-
-
PyVarObject Slots
-----------------
@@ -560,8 +547,8 @@ PyTypeObject Slots
Each slot has a section describing inheritance. If :c:func:`PyType_Ready`
may set a value when the field is set to ``NULL`` then there will also be
-a "Default" section. (Note that many fields set on :c:type:`PyBaseObject_Type`
-and :c:type:`PyType_Type` effectively act as defaults.)
+a "Default" section. (Note that many fields set on :c:data:`PyBaseObject_Type`
+and :c:data:`PyType_Type` effectively act as defaults.)
.. c:member:: const char* PyTypeObject.tp_name
@@ -570,7 +557,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
name, followed by a dot, followed by the type name; for built-in types, it
should be just the type name. If the module is a submodule of a package, the
full package name is part of the full module name. For example, a type named
- :class:`T` defined in module :mod:`M` in subpackage :mod:`Q` in package :mod:`P`
+ :class:`!T` defined in module :mod:`!M` in subpackage :mod:`!Q` in package :mod:`!P`
should have the :c:member:`~PyTypeObject.tp_name` initializer ``"P.Q.M.T"``.
For :ref:`dynamically allocated type objects `,
@@ -610,20 +597,20 @@ and :c:type:`PyType_Type` effectively act as defaults.)
instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`.
For a type with variable-length instances, the instances must have an
- :attr:`ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N
+ :c:member:`~PyVarObject.ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N
times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object. The value of
- N is typically stored in the instance's :attr:`ob_size` field. There are
- exceptions: for example, ints use a negative :attr:`ob_size` to indicate a
+ N is typically stored in the instance's :c:member:`~PyVarObject.ob_size` field. There are
+ exceptions: for example, ints use a negative :c:member:`~PyVarObject.ob_size` to indicate a
negative number, and N is ``abs(ob_size)`` there. Also, the presence of an
- :attr:`ob_size` field in the instance layout doesn't mean that the instance
+ :c:member:`~PyVarObject.ob_size` field in the instance layout doesn't mean that the instance
structure is variable-length (for example, the structure for the list type has
- fixed-length instances, yet those instances have a meaningful :attr:`ob_size`
+ fixed-length instances, yet those instances have a meaningful :c:member:`~PyVarObject.ob_size`
field).
The basic size includes the fields in the instance declared by the macro
:c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to
- declare the instance struct) and this in turn includes the :attr:`_ob_prev` and
- :attr:`_ob_next` fields if they are present. This means that the only correct
+ declare the instance struct) and this in turn includes the :c:member:`~PyObject._ob_prev` and
+ :c:member:`~PyObject._ob_next` fields if they are present. This means that the only correct
way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the
``sizeof`` operator on the struct used to declare the instance layout.
The basic size does not include the GC header size.
@@ -660,15 +647,15 @@ and :c:type:`PyType_Type` effectively act as defaults.)
memory buffers owned by the instance (using the freeing function corresponding
to the allocation function used to allocate the buffer), and call the type's
:c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable
- (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is
+ (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is
permissible to call the object deallocator directly instead of via
:c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the
instance; this is normally :c:func:`PyObject_Del` if the instance was allocated
- using :c:func:`PyObject_New` or :c:func:`PyObject_VarNew`, or
+ using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or
:c:func:`PyObject_GC_Del` if the instance was allocated using
- :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`.
+ :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`.
- If the type supports garbage collection (has the :const:`Py_TPFLAGS_HAVE_GC`
+ If the type supports garbage collection (has the :c:macro:`Py_TPFLAGS_HAVE_GC`
flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack`
before clearing any member fields.
@@ -680,8 +667,9 @@ and :c:type:`PyType_Type` effectively act as defaults.)
Py_TYPE(self)->tp_free((PyObject *)self);
}
- Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the
- deallocator should decrement the reference count for its type object after
+ Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the
+ deallocator should release the owned reference to its type object
+ (via :c:func:`Py_DECREF`) after
calling the type deallocator. In order to avoid dangling pointers, the
recommended way to achieve this is:
@@ -707,12 +695,12 @@ and :c:type:`PyType_Type` effectively act as defaults.)
a more efficient alternative
of the simpler :c:member:`~PyTypeObject.tp_call`.
- This field is only used if the flag :const:`Py_TPFLAGS_HAVE_VECTORCALL`
+ This field is only used if the flag :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL`
is set. If so, this must be a positive integer containing the offset in the
instance of a :c:type:`vectorcallfunc` pointer.
The *vectorcallfunc* pointer may be ``NULL``, in which case the instance behaves
- as if :const:`Py_TPFLAGS_HAVE_VECTORCALL` was not set: calling the instance
+ as if :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` was not set: calling the instance
falls back to :c:member:`~PyTypeObject.tp_call`.
Any class that sets ``Py_TPFLAGS_HAVE_VECTORCALL`` must also set
@@ -731,15 +719,15 @@ and :c:type:`PyType_Type` effectively act as defaults.)
Before version 3.12, it was not recommended for
:ref:`mutable heap types ` to implement the vectorcall
protocol.
- When a user sets :attr:`~type.__call__` in Python code, only *tp_call* is
+ When a user sets :attr:`~object.__call__` in Python code, only *tp_call* is
updated, likely making it inconsistent with the vectorcall function.
Since 3.12, setting ``__call__`` will disable vectorcall optimization
- by clearing the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag.
+ by clearing the :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag.
**Inheritance:**
This field is always inherited.
- However, the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag is not
+ However, the :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag is not
always inherited. If it's not set, then the subclass won't use
:ref:`vectorcall `, except when
:c:func:`PyVectorcall_Call` is explicitly called.
@@ -755,7 +743,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
- Group: :attr:`tp_getattr`, :attr:`tp_getattro`
+ Group: :c:member:`~PyTypeObject.tp_getattr`, :c:member:`~PyTypeObject.tp_getattro`
This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattro`: a subtype
inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when
@@ -772,7 +760,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
- Group: :attr:`tp_setattr`, :attr:`tp_setattro`
+ Group: :c:member:`~PyTypeObject.tp_setattr`, :c:member:`~PyTypeObject.tp_setattro`
This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattro`: a subtype
inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when
@@ -796,7 +784,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. c:member:: reprfunc PyTypeObject.tp_repr
- .. index:: builtin: repr
+ .. index:: pair: built-in function; repr
An optional pointer to a function that implements the built-in function
:func:`repr`.
@@ -861,7 +849,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. c:member:: hashfunc PyTypeObject.tp_hash
- .. index:: builtin: hash
+ .. index:: pair: built-in function; hash
An optional pointer to a function that implements the built-in function
:func:`hash`.
@@ -874,7 +862,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
normal return value; when an error occurs during the computation of the hash
value, the function should set an exception and return ``-1``.
- When this field is not set (*and* :attr:`tp_richcompare` is not set),
+ When this field is not set (*and* :c:member:`~PyTypeObject.tp_richcompare` is not set),
an attempt to take the hash of the object raises :exc:`TypeError`.
This is the same as setting it to :c:func:`PyObject_HashNotImplemented`.
@@ -888,7 +876,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
- Group: :attr:`tp_hash`, :attr:`tp_richcompare`
+ Group: :c:member:`~PyTypeObject.tp_hash`, :c:member:`~PyTypeObject.tp_richcompare`
This field is inherited by subtypes together with
:c:member:`~PyTypeObject.tp_richcompare`: a subtype inherits both of
@@ -947,7 +935,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
- Group: :attr:`tp_getattr`, :attr:`tp_getattro`
+ Group: :c:member:`~PyTypeObject.tp_getattr`, :c:member:`~PyTypeObject.tp_getattro`
This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattr`: a subtype
inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when
@@ -955,7 +943,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Default:**
- :c:type:`PyBaseObject_Type` uses :c:func:`PyObject_GenericGetAttr`.
+ :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericGetAttr`.
.. c:member:: setattrofunc PyTypeObject.tp_setattro
@@ -973,7 +961,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
- Group: :attr:`tp_setattr`, :attr:`tp_setattro`
+ Group: :c:member:`~PyTypeObject.tp_setattr`, :c:member:`~PyTypeObject.tp_setattro`
This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattr`: a subtype
inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when
@@ -981,7 +969,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Default:**
- :c:type:`PyBaseObject_Type` uses :c:func:`PyObject_GenericSetAttr`.
+ :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericSetAttr`.
.. c:member:: PyBufferProcs* PyTypeObject.tp_as_buffer
@@ -1013,30 +1001,32 @@ and :c:type:`PyType_Type` effectively act as defaults.)
this flag bit. The flag bits that pertain to extension structures are strictly
inherited if the extension structure is inherited, i.e. the base type's value of
the flag bit is copied into the subtype together with a pointer to the extension
- structure. The :const:`Py_TPFLAGS_HAVE_GC` flag bit is inherited together with
+ structure. The :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is inherited together with
the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields, i.e. if the
- :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the
+ :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the
:c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have
``NULL`` values.
.. XXX are most flag bits *really* inherited individually?
**Default:**
- :c:type:`PyBaseObject_Type` uses
+ :c:data:`PyBaseObject_Type` uses
``Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE``.
**Bit Masks:**
+ .. c:namespace:: NULL
+
The following bit masks are currently defined; these can be ORed together using
the ``|`` operator to form the value of the :c:member:`~PyTypeObject.tp_flags` field. The macro
:c:func:`PyType_HasFeature` takes a type and a flags value, *tp* and *f*, and
checks whether ``tp->tp_flags & f`` is non-zero.
- .. data:: Py_TPFLAGS_HEAPTYPE
+ .. c:macro:: Py_TPFLAGS_HEAPTYPE
This bit is set when the type object itself is allocated on the heap, for
example, types created dynamically using :c:func:`PyType_FromSpec`. In this
- case, the :attr:`ob_type` field of its instances is considered a reference to
+ case, the :c:member:`~PyObject.ob_type` field of its instances is considered a reference to
the type, and the type object is INCREF'ed when a new instance is created, and
DECREF'ed when an instance is destroyed (this does not apply to instances of
subtypes; only the type referenced by the instance's ob_type gets INCREF'ed or
@@ -1047,7 +1037,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
???
- .. data:: Py_TPFLAGS_BASETYPE
+ .. c:macro:: Py_TPFLAGS_BASETYPE
This bit is set when the type can be used as the base type of another type. If
this bit is clear, the type cannot be subtyped (similar to a "final" class in
@@ -1058,7 +1048,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
???
- .. data:: Py_TPFLAGS_READY
+ .. c:macro:: Py_TPFLAGS_READY
This bit is set when the type object has been fully initialized by
:c:func:`PyType_Ready`.
@@ -1068,7 +1058,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
???
- .. data:: Py_TPFLAGS_READYING
+ .. c:macro:: Py_TPFLAGS_READYING
This bit is set while :c:func:`PyType_Ready` is in the process of initializing
the type object.
@@ -1078,10 +1068,10 @@ and :c:type:`PyType_Type` effectively act as defaults.)
???
- .. data:: Py_TPFLAGS_HAVE_GC
+ .. c:macro:: Py_TPFLAGS_HAVE_GC
This bit is set when the object supports garbage collection. If this bit
- is set, instances must be created using :c:func:`PyObject_GC_New` and
+ is set, instances must be created using :c:macro:`PyObject_GC_New` and
destroyed using :c:func:`PyObject_GC_Del`. More information in section
:ref:`supporting-cycle-detection`. This bit also implies that the
GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in
@@ -1089,28 +1079,28 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
- Group: :const:`Py_TPFLAGS_HAVE_GC`, :attr:`tp_traverse`, :attr:`tp_clear`
+ Group: :c:macro:`Py_TPFLAGS_HAVE_GC`, :c:member:`~PyTypeObject.tp_traverse`, :c:member:`~PyTypeObject.tp_clear`
- The :const:`Py_TPFLAGS_HAVE_GC` flag bit is inherited
- together with the :attr:`tp_traverse` and :attr:`tp_clear`
- fields, i.e. if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is
- clear in the subtype and the :attr:`tp_traverse` and
- :attr:`tp_clear` fields in the subtype exist and have ``NULL``
+ The :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is inherited
+ together with the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear`
+ fields, i.e. if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is
+ clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and
+ :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have ``NULL``
values.
- .. data:: Py_TPFLAGS_DEFAULT
+ .. c:macro:: Py_TPFLAGS_DEFAULT
This is a bitmask of all the bits that pertain to the existence of certain
fields in the type object and its extension structures. Currently, it includes
- the following bits: :const:`Py_TPFLAGS_HAVE_STACKLESS_EXTENSION`.
+ the following bits: :c:macro:`Py_TPFLAGS_HAVE_STACKLESS_EXTENSION`.
**Inheritance:**
???
- .. data:: Py_TPFLAGS_METHOD_DESCRIPTOR
+ .. c:macro:: Py_TPFLAGS_METHOD_DESCRIPTOR
This bit indicates that objects behave like unbound methods.
@@ -1131,17 +1121,20 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
This flag is never inherited by types without the
- :const:`Py_TPFLAGS_IMMUTABLETYPE` flag set. For extension types, it is
+ :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag set. For extension types, it is
inherited whenever :c:member:`~PyTypeObject.tp_descr_get` is inherited.
- .. data:: Py_TPFLAGS_MANAGED_DICT
+ .. c:macro:: Py_TPFLAGS_MANAGED_DICT
- This bit indicates that instances of the class have a ``__dict___``
- attribute, and that the space for the dictionary is managed by the VM.
+ This bit indicates that instances of the class have a ``__dict__``
+ attribute, and that the space for the dictionary is managed by the VM.
- If this flag is set, :const:`Py_TPFLAGS_HAVE_GC` should also be set.
+ If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set.
- .. versionadded:: 3.12
+ The type traverse function must call :c:func:`PyObject_VisitManagedDict`
+ and its clear function must call :c:func:`PyObject_ClearManagedDict`.
+
+ .. versionadded:: 3.12
**Inheritance:**
@@ -1149,12 +1142,12 @@ and :c:type:`PyType_Type` effectively act as defaults.)
:c:member:`~PyTypeObject.tp_dictoffset` field is set in a superclass.
- .. data:: Py_TPFLAGS_MANAGED_WEAKREF
+ .. c:macro:: Py_TPFLAGS_MANAGED_WEAKREF
- This bit indicates that instances of the class should be weakly
- referenceable.
+ This bit indicates that instances of the class should be weakly
+ referenceable.
- .. versionadded:: 3.12
+ .. versionadded:: 3.12
**Inheritance:**
@@ -1162,17 +1155,37 @@ and :c:type:`PyType_Type` effectively act as defaults.)
:c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass.
+ .. c:macro:: Py_TPFLAGS_ITEMS_AT_END
+
+ Only usable with variable-size types, i.e. ones with non-zero
+ :c:member:`~PyTypeObject.tp_itemsize`.
+
+ Indicates that the variable-sized portion of an instance of this type is
+ at the end of the instance's memory area, at an offset of
+ ``Py_TYPE(obj)->tp_basicsize`` (which may be different in each
+ subclass).
+
+ When setting this flag, be sure that all superclasses either
+ use this memory layout, or are not variable-sized.
+ Python does not check this.
+
+ .. versionadded:: 3.12
+
+ **Inheritance:**
+
+ This flag is inherited.
+
.. XXX Document more flags here?
- .. data:: Py_TPFLAGS_LONG_SUBCLASS
- .. data:: Py_TPFLAGS_LIST_SUBCLASS
- .. data:: Py_TPFLAGS_TUPLE_SUBCLASS
- .. data:: Py_TPFLAGS_BYTES_SUBCLASS
- .. data:: Py_TPFLAGS_UNICODE_SUBCLASS
- .. data:: Py_TPFLAGS_DICT_SUBCLASS
- .. data:: Py_TPFLAGS_BASE_EXC_SUBCLASS
- .. data:: Py_TPFLAGS_TYPE_SUBCLASS
+ .. c:macro:: Py_TPFLAGS_LONG_SUBCLASS
+ .. c:macro:: Py_TPFLAGS_LIST_SUBCLASS
+ .. c:macro:: Py_TPFLAGS_TUPLE_SUBCLASS
+ .. c:macro:: Py_TPFLAGS_BYTES_SUBCLASS
+ .. c:macro:: Py_TPFLAGS_UNICODE_SUBCLASS
+ .. c:macro:: Py_TPFLAGS_DICT_SUBCLASS
+ .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS
+ .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS
These flags are used by functions such as
:c:func:`PyLong_Check` to quickly determine if a type is a subclass
@@ -1183,7 +1196,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
will behave differently depending on what kind of check is used.
- .. data:: Py_TPFLAGS_HAVE_FINALIZE
+ .. c:macro:: Py_TPFLAGS_HAVE_FINALIZE
This bit is set when the :c:member:`~PyTypeObject.tp_finalize` slot is present in the
type structure.
@@ -1196,7 +1209,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
type structure.
- .. data:: Py_TPFLAGS_HAVE_VECTORCALL
+ .. c:macro:: Py_TPFLAGS_HAVE_VECTORCALL
This bit is set when the class implements
the :ref:`vectorcall protocol `.
@@ -1216,7 +1229,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
This flag can now be inherited by mutable classes.
- .. data:: Py_TPFLAGS_IMMUTABLETYPE
+ .. c:macro:: Py_TPFLAGS_IMMUTABLETYPE
This bit is set for type objects that are immutable: type attributes cannot be set nor deleted.
@@ -1229,7 +1242,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. versionadded:: 3.10
- .. data:: Py_TPFLAGS_DISALLOW_INSTANTIATION
+ .. c:macro:: Py_TPFLAGS_DISALLOW_INSTANTIATION
Disallow creating instances of the type: set
:c:member:`~PyTypeObject.tp_new` to NULL and don't create the ``__new__``
@@ -1245,11 +1258,22 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
This flag is not inherited.
+ However, subclasses will not be instantiable unless they provide a
+ non-NULL :c:member:`~PyTypeObject.tp_new` (which is only possible
+ via the C API).
+
+ .. note::
+
+ To disallow instantiating a class directly but allow instantiating
+ its subclasses (e.g. for an :term:`abstract base class`),
+ do not use this flag.
+ Instead, make :c:member:`~PyTypeObject.tp_new` only succeed for
+ subclasses.
.. versionadded:: 3.10
- .. data:: Py_TPFLAGS_MAPPING
+ .. c:macro:: Py_TPFLAGS_MAPPING
This bit indicates that instances of the class may match mapping patterns
when used as the subject of a :keyword:`match` block. It is automatically
@@ -1258,20 +1282,20 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. note::
- :const:`Py_TPFLAGS_MAPPING` and :const:`Py_TPFLAGS_SEQUENCE` are
+ :c:macro:`Py_TPFLAGS_MAPPING` and :c:macro:`Py_TPFLAGS_SEQUENCE` are
mutually exclusive; it is an error to enable both flags simultaneously.
**Inheritance:**
This flag is inherited by types that do not already set
- :const:`Py_TPFLAGS_SEQUENCE`.
+ :c:macro:`Py_TPFLAGS_SEQUENCE`.
.. seealso:: :pep:`634` -- Structural Pattern Matching: Specification
.. versionadded:: 3.10
- .. data:: Py_TPFLAGS_SEQUENCE
+ .. c:macro:: Py_TPFLAGS_SEQUENCE
This bit indicates that instances of the class may match sequence patterns
when used as the subject of a :keyword:`match` block. It is automatically
@@ -1280,19 +1304,29 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. note::
- :const:`Py_TPFLAGS_MAPPING` and :const:`Py_TPFLAGS_SEQUENCE` are
+ :c:macro:`Py_TPFLAGS_MAPPING` and :c:macro:`Py_TPFLAGS_SEQUENCE` are
mutually exclusive; it is an error to enable both flags simultaneously.
**Inheritance:**
This flag is inherited by types that do not already set
- :const:`Py_TPFLAGS_MAPPING`.
+ :c:macro:`Py_TPFLAGS_MAPPING`.
.. seealso:: :pep:`634` -- Structural Pattern Matching: Specification
.. versionadded:: 3.10
+ .. c:macro:: Py_TPFLAGS_VALID_VERSION_TAG
+
+ Internal. Do not set or unset this flag.
+ To indicate that a class has changed call :c:func:`PyType_Modified`
+
+ .. warning::
+ This flag is present in header files, but is an internal feature and should
+ not be used. It will be removed in a future version of CPython
+
+
.. c:member:: const char* PyTypeObject.tp_doc
An optional pointer to a NUL-terminated C string giving the docstring for this
@@ -1307,7 +1341,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. c:member:: traverseproc PyTypeObject.tp_traverse
An optional pointer to a traversal function for the garbage collector. This is
- only used if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is::
+ only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is::
int tp_traverse(PyObject *self, visitproc visit, void *arg);
@@ -1317,8 +1351,8 @@ and :c:type:`PyType_Type` effectively act as defaults.)
The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect
reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function
simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python
- objects that the instance owns. For example, this is function :c:func:`local_traverse` from the
- :mod:`_thread` extension module::
+ objects that the instance owns. For example, this is function :c:func:`!local_traverse` from the
+ :mod:`!_thread` extension module::
static int
local_traverse(localobject *self, visitproc visit, void *arg)
@@ -1337,6 +1371,23 @@ and :c:type:`PyType_Type` effectively act as defaults.)
debugging aid you may want to visit it anyway just so the :mod:`gc` module's
:func:`~gc.get_referents` function will include it.
+ Heap types (:c:macro:`Py_TPFLAGS_HEAPTYPE`) must visit their type with::
+
+ Py_VISIT(Py_TYPE(self));
+
+ It is only needed since Python 3.9. To support Python 3.8 and older, this
+ line must be conditionnal::
+
+ #if PY_VERSION_HEX >= 0x03090000
+ Py_VISIT(Py_TYPE(self));
+ #endif
+
+ If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the
+ :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call
+ :c:func:`PyObject_VisitManagedDict` like this::
+
+ PyObject_VisitManagedDict((PyObject*)self, visit, arg);
+
.. warning::
When implementing :c:member:`~PyTypeObject.tp_traverse`, only the
members that the instance *owns* (by having :term:`strong references
@@ -1350,7 +1401,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
are allowed to be removed even if the instance is still alive).
Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to
- :c:func:`local_traverse` to have these specific names; don't name them just
+ :c:func:`!local_traverse` to have these specific names; don't name them just
anything.
Instances of :ref:`heap-allocated types ` hold a reference to
@@ -1369,10 +1420,10 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
- Group: :const:`Py_TPFLAGS_HAVE_GC`, :attr:`tp_traverse`, :attr:`tp_clear`
+ Group: :c:macro:`Py_TPFLAGS_HAVE_GC`, :c:member:`~PyTypeObject.tp_traverse`, :c:member:`~PyTypeObject.tp_clear`
This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_clear` and the
- :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and
+ :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and
:c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in
the subtype.
@@ -1380,7 +1431,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. c:member:: inquiry PyTypeObject.tp_clear
An optional pointer to a clear function for the garbage collector. This is only
- used if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is::
+ used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is::
int tp_clear(PyObject *);
@@ -1409,9 +1460,10 @@ and :c:type:`PyType_Type` effectively act as defaults.)
}
The :c:func:`Py_CLEAR` macro should be used, because clearing references is
- delicate: the reference to the contained object must not be decremented until
+ delicate: the reference to the contained object must not be released
+ (via :c:func:`Py_DECREF`) until
after the pointer to the contained object is set to ``NULL``. This is because
- decrementing the reference count may cause the contained object to become trash,
+ releasing the reference may cause the contained object to become trash,
triggering a chain of reclamation activity that may include invoking arbitrary
Python code (due to finalizers, or weakref callbacks, associated with the
contained object). If it's possible for such code to reference *self* again,
@@ -1419,6 +1471,12 @@ and :c:type:`PyType_Type` effectively act as defaults.)
so that *self* knows the contained object can no longer be used. The
:c:func:`Py_CLEAR` macro performs the operations in a safe order.
+ If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the
+ :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call
+ :c:func:`PyObject_ClearManagedDict` like this::
+
+ PyObject_ClearManagedDict((PyObject*)self);
+
Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called
before an instance is deallocated. For example, when reference counting
is enough to determine that an object is no longer used, the cyclic garbage
@@ -1436,10 +1494,10 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
- Group: :const:`Py_TPFLAGS_HAVE_GC`, :attr:`tp_traverse`, :attr:`tp_clear`
+ Group: :c:macro:`Py_TPFLAGS_HAVE_GC`, :c:member:`~PyTypeObject.tp_traverse`, :c:member:`~PyTypeObject.tp_clear`
This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_traverse` and the
- :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and
+ :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and
:c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in
the subtype.
@@ -1461,21 +1519,23 @@ and :c:type:`PyType_Type` effectively act as defaults.)
The following constants are defined to be used as the third argument for
:c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`:
- +----------------+------------+
- | Constant | Comparison |
- +================+============+
- | :const:`Py_LT` | ``<`` |
- +----------------+------------+
- | :const:`Py_LE` | ``<=`` |
- +----------------+------------+
- | :const:`Py_EQ` | ``==`` |
- +----------------+------------+
- | :const:`Py_NE` | ``!=`` |
- +----------------+------------+
- | :const:`Py_GT` | ``>`` |
- +----------------+------------+
- | :const:`Py_GE` | ``>=`` |
- +----------------+------------+
+ .. c:namespace:: NULL
+
+ +--------------------+------------+
+ | Constant | Comparison |
+ +====================+============+
+ | .. c:macro:: Py_LT | ``<`` |
+ +--------------------+------------+
+ | .. c:macro:: Py_LE | ``<=`` |
+ +--------------------+------------+
+ | .. c:macro:: Py_EQ | ``==`` |
+ +--------------------+------------+
+ | .. c:macro:: Py_NE | ``!=`` |
+ +--------------------+------------+
+ | .. c:macro:: Py_GT | ``>`` |
+ +--------------------+------------+
+ | .. c:macro:: Py_GE | ``>=`` |
+ +--------------------+------------+
The following macro is defined to ease writing rich comparison functions:
@@ -1487,7 +1547,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
they may be C ints or floats). The third argument specifies the requested
operation, as for :c:func:`PyObject_RichCompare`.
- The return value's reference count is properly incremented.
+ The returned value is a new :term:`strong reference`.
On error, sets an exception and returns ``NULL`` from the function.
@@ -1495,7 +1555,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Inheritance:**
- Group: :attr:`tp_hash`, :attr:`tp_richcompare`
+ Group: :c:member:`~PyTypeObject.tp_hash`, :c:member:`~PyTypeObject.tp_richcompare`
This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_hash`:
a subtype inherits :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` when
@@ -1504,16 +1564,16 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Default:**
- :c:type:`PyBaseObject_Type` provides a :attr:`tp_richcompare`
+ :c:data:`PyBaseObject_Type` provides a :c:member:`~PyTypeObject.tp_richcompare`
implementation, which may be inherited. However, if only
- :attr:`tp_hash` is defined, not even the inherited function is used
+ :c:member:`~PyTypeObject.tp_hash` is defined, not even the inherited function is used
and instances of the type will not be able to participate in any
comparisons.
.. c:member:: Py_ssize_t PyTypeObject.tp_weaklistoffset
- While this field is still supported, :const:`Py_TPFLAGS_MANAGED_WEAKREF`
+ While this field is still supported, :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF`
should be used instead, if at all possible.
If the instances of this type are weakly referenceable, this field is greater
@@ -1526,7 +1586,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for
weak references to the type object itself.
- It is an error to set both the :const:`Py_TPFLAGS_MANAGED_WEAKREF` bit and
+ It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and
:c:member:`~PyTypeObject.tp_weaklist`.
**Inheritance:**
@@ -1538,7 +1598,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Default:**
- If the :const:`Py_TPFLAGS_MANAGED_WEAKREF` bit is set in the
+ If the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit is set in the
:c:member:`~PyTypeObject.tp_dict` field, then
:c:member:`~PyTypeObject.tp_weaklistoffset` will be set to a negative value,
to indicate that it is unsafe to use this field.
@@ -1641,7 +1701,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
to a pointer, are valid C99 address constants.
However, the unary '&' operator applied to a non-static variable
- like :c:func:`PyBaseObject_Type` is not required to produce an address
+ like :c:data:`PyBaseObject_Type` is not required to produce an address
constant. Compilers may support this (gcc does), MSVC does not.
Both compilers are strictly standard conforming in this particular
behavior.
@@ -1667,7 +1727,19 @@ and :c:type:`PyType_Type` effectively act as defaults.)
called; it may also be initialized to a dictionary containing initial attributes
for the type. Once :c:func:`PyType_Ready` has initialized the type, extra
attributes for the type may be added to this dictionary only if they don't
- correspond to overloaded operations (like :meth:`__add__`).
+ correspond to overloaded operations (like :meth:`~object.__add__`). Once
+ initialization for the type has finished, this field should be
+ treated as read-only.
+
+ Some types may not store their dictionary in this slot.
+ Use :c:func:`PyType_GetDict` to retrieve the dictionary for an arbitrary
+ type.
+
+ .. versionchanged:: 3.12
+
+ Internals detail: For static builtin types, this is always ``NULL``.
+ Instead, the dict for such types is stored on ``PyInterpreterState``.
+ Use :c:func:`PyType_GetDict` to get the dict for an arbitrary type.
**Inheritance:**
@@ -1720,7 +1792,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. c:member:: Py_ssize_t PyTypeObject.tp_dictoffset
- While this field is still supported, :const:`Py_TPFLAGS_MANAGED_DICT` should be
+ While this field is still supported, :c:macro:`Py_TPFLAGS_MANAGED_DICT` should be
used instead, if at all possible.
If the instances of this type have a dictionary containing instance variables,
@@ -1739,7 +1811,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
dictionary, so it is may be more efficient to call :c:func:`PyObject_GetAttr`
when accessing an attribute on the object.
- It is an error to set both the :const:`Py_TPFLAGS_MANAGED_WEAKREF` bit and
+ It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and
:c:member:`~PyTypeObject.tp_dictoffset`.
**Inheritance:**
@@ -1747,15 +1819,15 @@ and :c:type:`PyType_Type` effectively act as defaults.)
This field is inherited by subtypes. A subtype should not override this offset;
doing so could be unsafe, if C code tries to access the dictionary at the
previous offset.
- To properly support inheritance, use :const:`Py_TPFLAGS_MANAGED_DICT`.
+ To properly support inheritance, use :c:macro:`Py_TPFLAGS_MANAGED_DICT`.
**Default:**
This slot has no default. For :ref:`static types `, if the
- field is ``NULL`` then no :attr:`__dict__` gets created for instances.
+ field is ``NULL`` then no :attr:`~object.__dict__` gets created for instances.
- If the :const:`Py_TPFLAGS_MANAGED_DICT` bit is set in the
- :c:member:`~PyTypeObject.tp_dict` field, then
+ If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the
+ :c:member:`~PyTypeObject.tp_flags` field, then
:c:member:`~PyTypeObject.tp_dictoffset` will be set to ``-1``, to indicate
that it is unsafe to use this field.
@@ -1764,10 +1836,10 @@ and :c:type:`PyType_Type` effectively act as defaults.)
An optional pointer to an instance initialization function.
- This function corresponds to the :meth:`__init__` method of classes. Like
- :meth:`__init__`, it is possible to create an instance without calling
- :meth:`__init__`, and it is possible to reinitialize an instance by calling its
- :meth:`__init__` method again.
+ This function corresponds to the :meth:`~object.__init__` method of classes. Like
+ :meth:`!__init__`, it is possible to create an instance without calling
+ :meth:`!__init__`, and it is possible to reinitialize an instance by calling its
+ :meth:`!__init__` method again.
The function signature is::
@@ -1775,7 +1847,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
The self argument is the instance to be initialized; the *args* and *kwds*
arguments represent positional and keyword arguments of the call to
- :meth:`__init__`.
+ :meth:`~object.__init__`.
The :c:member:`~PyTypeObject.tp_init` function, if not ``NULL``, is called when an instance is
created normally by calling its type, after the type's :c:member:`~PyTypeObject.tp_new` function
@@ -1814,7 +1886,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
:c:func:`PyType_GenericAlloc`, to force a standard heap
allocation strategy.
- For static subtypes, :c:type:`PyBaseObject_Type` uses
+ For static subtypes, :c:data:`PyBaseObject_Type` uses
:c:func:`PyType_GenericAlloc`. That is the recommended value
for all statically defined types.
@@ -1841,7 +1913,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
in :c:member:`~PyTypeObject.tp_new`, while for mutable types, most initialization should be
deferred to :c:member:`~PyTypeObject.tp_init`.
- Set the :const:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag to disallow creating
+ Set the :c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag to disallow creating
instances of the type in Python.
**Inheritance:**
@@ -1875,9 +1947,9 @@ and :c:type:`PyType_Type` effectively act as defaults.)
In dynamic subtypes, this field is set to a deallocator suitable to
match :c:func:`PyType_GenericAlloc` and the value of the
- :const:`Py_TPFLAGS_HAVE_GC` flag bit.
+ :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit.
- For static subtypes, :c:type:`PyBaseObject_Type` uses PyObject_Del.
+ For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Del`.
.. c:member:: inquiry PyTypeObject.tp_is_gc
@@ -1886,7 +1958,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
The garbage collector needs to know whether a particular object is collectible
or not. Normally, it is sufficient to look at the object's type's
- :c:member:`~PyTypeObject.tp_flags` field, and check the :const:`Py_TPFLAGS_HAVE_GC` flag bit. But
+ :c:member:`~PyTypeObject.tp_flags` field, and check the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. But
some types have a mixture of statically and dynamically allocated instances, and
the statically allocated instances are not collectible. Such types should
define this function; it should return ``1`` for a collectible instance, and
@@ -1905,15 +1977,26 @@ and :c:type:`PyType_Type` effectively act as defaults.)
**Default:**
This slot has no default. If this field is ``NULL``,
- :const:`Py_TPFLAGS_HAVE_GC` is used as the functional equivalent.
+ :c:macro:`Py_TPFLAGS_HAVE_GC` is used as the functional equivalent.
.. c:member:: PyObject* PyTypeObject.tp_bases
Tuple of base types.
- This is set for types created by a class statement. It should be ``NULL`` for
- statically defined types.
+ This field should be set to ``NULL`` and treated as read-only.
+ Python will fill it in when the type is :c:func:`initialized `.
+
+ For dynamically created classes, the ``Py_tp_bases``
+ :c:type:`slot ` can be used instead of the *bases* argument
+ of :c:func:`PyType_FromSpecWithBases`.
+ The argument form is preferred.
+
+ .. warning::
+
+ Multiple inheritance does not work well for statically defined types.
+ If you set ``tp_bases`` to a tuple, Python will not raise an error,
+ but some slots will only be inherited from the first base.
**Inheritance:**
@@ -1925,6 +2008,8 @@ and :c:type:`PyType_Type` effectively act as defaults.)
Tuple containing the expanded set of base types, starting with the type itself
and ending with :class:`object`, in Method Resolution Order.
+ This field should be set to ``NULL`` and treated as read-only.
+ Python will fill it in when the type is :c:func:`initialized `.
**Inheritance:**
@@ -2039,7 +2124,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. versionchanged:: 3.8
Before version 3.8 it was necessary to set the
- :const:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be
+ :c:macro:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be
used. This is no longer required.
.. seealso:: "Safe object finalization" (:pep:`442`)
@@ -2051,7 +2136,7 @@ and :c:type:`PyType_Type` effectively act as defaults.)
In other words, it is used to implement
:ref:`vectorcall ` for ``type.__call__``.
If ``tp_vectorcall`` is ``NULL``, the default call implementation
- using :attr:`__new__` and :attr:`__init__` is used.
+ using :meth:`~object.__new__` and :meth:`~object.__init__` is used.
**Inheritance:**
@@ -2060,6 +2145,13 @@ and :c:type:`PyType_Type` effectively act as defaults.)
.. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9)
+.. c:member:: unsigned char PyTypeObject.tp_watched
+
+ Internal. Do not use.
+
+ .. versionadded:: 3.12
+
+
.. _static-types:
Static Types
@@ -2080,7 +2172,7 @@ This results in types that are limited relative to types defined in Python:
include any subinterpreter-specific state.
Also, since :c:type:`PyTypeObject` is only part of the :ref:`Limited API
-` as an opaque struct, any extension modules using static types must be
+` as an opaque struct, any extension modules using static types must be
compiled for a specific Python minor version.
@@ -2091,7 +2183,7 @@ Heap Types
An alternative to :ref:`static types ` is *heap-allocated types*,
or *heap types* for short, which correspond closely to classes created by
-Python's ``class`` statement. Heap types have the :const:`Py_TPFLAGS_HEAPTYPE`
+Python's ``class`` statement. Heap types have the :c:macro:`Py_TPFLAGS_HEAPTYPE`
flag set.
This is done by filling a :c:type:`PyType_Spec` structure and calling
@@ -2171,8 +2263,8 @@ Number Object Structures
.. note::
- The :c:data:`nb_reserved` field should always be ``NULL``. It
- was previously called :c:data:`nb_long`, and was renamed in
+ The :c:member:`~PyNumberMethods.nb_reserved` field should always be ``NULL``. It
+ was previously called :c:member:`!nb_long`, and was renamed in
Python 3.0.1.
.. c:member:: binaryfunc PyNumberMethods.nb_add
@@ -2243,8 +2335,8 @@ Mapping Object Structures
.. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript
This function is used by :c:func:`PyObject_SetItem`,
- :c:func:`PyObject_DelItem`, :c:func:`PyObject_SetSlice` and
- :c:func:`PyObject_DelSlice`. It has the same signature as
+ :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and
+ :c:func:`PySequence_DelSlice`. It has the same signature as
:c:func:`!PyObject_SetItem`, but *v* can also be set to ``NULL`` to delete
an item. If this slot is ``NULL``, the object does not support item
assignment and deletion.
@@ -2290,9 +2382,9 @@ Sequence Object Structures
This slot must be filled for the :c:func:`PySequence_Check`
function to return ``1``, it can be ``NULL`` otherwise.
- Negative indexes are handled as follows: if the :attr:`sq_length` slot is
+ Negative indexes are handled as follows: if the :c:member:`~PySequenceMethods.sq_length` slot is
filled, it is called and the sequence length is used to compute a positive
- index which is passed to :attr:`sq_item`. If :attr:`sq_length` is ``NULL``,
+ index which is passed to :c:member:`~PySequenceMethods.sq_item`. If :c:member:`!sq_length` is ``NULL``,
the index is passed as is to the function.
.. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item
@@ -2355,7 +2447,7 @@ Buffer Object Structures
Except for point (3), an implementation of this function MUST take these
steps:
- (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`,
+ (1) Check if the request can be met. If not, raise :exc:`BufferError`,
set :c:expr:`view->obj` to ``NULL`` and return ``-1``.
(2) Fill in the requested fields.
@@ -2466,7 +2558,7 @@ Async Object Structures
PyObject *am_aiter(PyObject *self);
Must return an :term:`asynchronous iterator` object.
- See :meth:`__anext__` for details.
+ See :meth:`~object.__anext__` for details.
This slot may be set to ``NULL`` if an object does not implement
asynchronous iteration protocol.
@@ -2477,7 +2569,8 @@ Async Object Structures
PyObject *am_anext(PyObject *self);
- Must return an :term:`awaitable` object. See :meth:`__anext__` for details.
+ Must return an :term:`awaitable` object.
+ See :meth:`~object.__anext__` for details.
This slot may be set to ``NULL``.
.. c:member:: sendfunc PyAsyncMethods.am_send
@@ -2502,8 +2595,8 @@ Slot Type typedefs
The purpose of this function is to separate memory allocation from memory
initialization. It should return a pointer to a block of memory of adequate
length for the instance, suitably aligned, and initialized to zeros, but with
- :attr:`ob_refcnt` set to ``1`` and :attr:`ob_type` set to the type argument. If
- the type's :c:member:`~PyTypeObject.tp_itemsize` is non-zero, the object's :attr:`ob_size` field
+ :c:member:`~PyObject.ob_refcnt` set to ``1`` and :c:member:`~PyObject.ob_type` set to the type argument. If
+ the type's :c:member:`~PyTypeObject.tp_itemsize` is non-zero, the object's :c:member:`~PyVarObject.ob_size` field
should be initialized to *nitems* and the length of the allocated memory block
should be ``tp_basicsize + nitems*tp_itemsize``, rounded up to a multiple of
``sizeof(void*)``; otherwise, *nitems* is not used and the length of the block
@@ -2594,7 +2687,7 @@ Slot Type typedefs
.. c:type:: PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t)
-.. c:type:: int (*ssizeobjargproc)(PyObject *, Py_ssize_t)
+.. c:type:: int (*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *)
.. c:type:: int (*objobjproc)(PyObject *, PyObject *)
@@ -2699,7 +2792,7 @@ A type that supports weakrefs, instance dicts, and hashing::
A str subclass that cannot be subclassed and cannot be called
to create instances (e.g. uses a separate factory func) using
-:c:data:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag::
+:c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag::
typedef struct {
PyUnicodeObject raw;
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index f062f14e9a75614..e654412965a727d 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -44,7 +44,7 @@ Python:
.. c:type:: Py_UNICODE
- This is a typedef of :c:expr:`wchar_t`, which is a 16-bit type or 32-bit type
+ This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type
depending on the platform.
.. versionchanged:: 3.3
@@ -52,6 +52,8 @@ Python:
whether you selected a "narrow" or "wide" Unicode version of Python at
build time.
+ .. deprecated-removed:: 3.13 3.15
+
.. c:type:: PyASCIIObject
PyCompactUnicodeObject
@@ -270,25 +272,16 @@ These APIs can be used for fast direct character conversions:
Return the character *ch* converted to lower case.
- .. deprecated:: 3.3
- This function uses simple case mappings.
-
.. c:function:: Py_UCS4 Py_UNICODE_TOUPPER(Py_UCS4 ch)
Return the character *ch* converted to upper case.
- .. deprecated:: 3.3
- This function uses simple case mappings.
-
.. c:function:: Py_UCS4 Py_UNICODE_TOTITLE(Py_UCS4 ch)
Return the character *ch* converted to title case.
- .. deprecated:: 3.3
- This function uses simple case mappings.
-
.. c:function:: int Py_UNICODE_TODECIMAL(Py_UCS4 ch)
@@ -394,98 +387,149 @@ APIs:
arguments, calculate the size of the resulting Python Unicode string and return
a string with the values formatted into it. The variable arguments must be C
types and must correspond exactly to the format characters in the *format*
- ASCII-encoded string. The following format characters are allowed:
-
- .. % This should be exactly the same as the table in PyErr_Format.
-
- .. tabularcolumns:: |l|l|L|
-
- +-------------------+---------------------+----------------------------------+
- | Format Characters | Type | Comment |
- +===================+=====================+==================================+
- | :attr:`%%` | *n/a* | The literal % character. |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%c` | int | A single character, |
- | | | represented as a C int. |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%d` | int | Equivalent to |
- | | | ``printf("%d")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%u` | unsigned int | Equivalent to |
- | | | ``printf("%u")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%ld` | long | Equivalent to |
- | | | ``printf("%ld")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%li` | long | Equivalent to |
- | | | ``printf("%li")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%lu` | unsigned long | Equivalent to |
- | | | ``printf("%lu")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%lld` | long long | Equivalent to |
- | | | ``printf("%lld")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%lli` | long long | Equivalent to |
- | | | ``printf("%lli")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%llu` | unsigned long long | Equivalent to |
- | | | ``printf("%llu")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%zd` | :c:type:`\ | Equivalent to |
- | | Py_ssize_t` | ``printf("%zd")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%zi` | :c:type:`\ | Equivalent to |
- | | Py_ssize_t` | ``printf("%zi")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%zu` | size_t | Equivalent to |
- | | | ``printf("%zu")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%i` | int | Equivalent to |
- | | | ``printf("%i")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%x` | int | Equivalent to |
- | | | ``printf("%x")``. [1]_ |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%s` | const char\* | A null-terminated C character |
- | | | array. |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%p` | const void\* | The hex representation of a C |
- | | | pointer. Mostly equivalent to |
- | | | ``printf("%p")`` except that |
- | | | it is guaranteed to start with |
- | | | the literal ``0x`` regardless |
- | | | of what the platform's |
- | | | ``printf`` yields. |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%A` | PyObject\* | The result of calling |
- | | | :func:`ascii`. |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%U` | PyObject\* | A Unicode object. |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%V` | PyObject\*, | A Unicode object (which may be |
- | | const char\* | ``NULL``) and a null-terminated |
- | | | C character array as a second |
- | | | parameter (which will be used, |
- | | | if the first parameter is |
- | | | ``NULL``). |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%S` | PyObject\* | The result of calling |
- | | | :c:func:`PyObject_Str`. |
- +-------------------+---------------------+----------------------------------+
- | :attr:`%R` | PyObject\* | The result of calling |
- | | | :c:func:`PyObject_Repr`. |
- +-------------------+---------------------+----------------------------------+
+ ASCII-encoded string.
+
+ A conversion specifier contains two or more characters and has the following
+ components, which must occur in this order:
+
+ #. The ``'%'`` character, which marks the start of the specifier.
+
+ #. Conversion flags (optional), which affect the result of some conversion
+ types.
+
+ #. Minimum field width (optional).
+ If specified as an ``'*'`` (asterisk), the actual width is given in the
+ next argument, which must be of type :c:expr:`int`, and the object to
+ convert comes after the minimum field width and optional precision.
+
+ #. Precision (optional), given as a ``'.'`` (dot) followed by the precision.
+ If specified as ``'*'`` (an asterisk), the actual precision is given in
+ the next argument, which must be of type :c:expr:`int`, and the value to
+ convert comes after the precision.
+
+ #. Length modifier (optional).
+
+ #. Conversion type.
+
+ The conversion flag characters are:
+
+ .. tabularcolumns:: |l|L|
+
+ +-------+-------------------------------------------------------------+
+ | Flag | Meaning |
+ +=======+=============================================================+
+ | ``0`` | The conversion will be zero padded for numeric values. |
+ +-------+-------------------------------------------------------------+
+ | ``-`` | The converted value is left adjusted (overrides the ``0`` |
+ | | flag if both are given). |
+ +-------+-------------------------------------------------------------+
+
+ The length modifiers for following integer conversions (``d``, ``i``,
+ ``o``, ``u``, ``x``, or ``X``) specify the type of the argument
+ (:c:expr:`int` by default):
+
+ .. tabularcolumns:: |l|L|
+
+ +----------+-----------------------------------------------------+
+ | Modifier | Types |
+ +==========+=====================================================+
+ | ``l`` | :c:expr:`long` or :c:expr:`unsigned long` |
+ +----------+-----------------------------------------------------+
+ | ``ll`` | :c:expr:`long long` or :c:expr:`unsigned long long` |
+ +----------+-----------------------------------------------------+
+ | ``j`` | :c:type:`intmax_t` or :c:type:`uintmax_t` |
+ +----------+-----------------------------------------------------+
+ | ``z`` | :c:type:`size_t` or :c:type:`ssize_t` |
+ +----------+-----------------------------------------------------+
+ | ``t`` | :c:type:`ptrdiff_t` |
+ +----------+-----------------------------------------------------+
+
+ The length modifier ``l`` for following conversions ``s`` or ``V`` specify
+ that the type of the argument is :c:expr:`const wchar_t*`.
+
+ The conversion specifiers are:
+
+ .. list-table::
+ :widths: auto
+ :header-rows: 1
+
+ * - Conversion Specifier
+ - Type
+ - Comment
+
+ * - ``%``
+ - *n/a*
+ - The literal ``%`` character.
+
+ * - ``d``, ``i``
+ - Specified by the length modifier
+ - The decimal representation of a signed C integer.
+
+ * - ``u``
+ - Specified by the length modifier
+ - The decimal representation of an unsigned C integer.
+
+ * - ``o``
+ - Specified by the length modifier
+ - The octal representation of an unsigned C integer.
+
+ * - ``x``
+ - Specified by the length modifier
+ - The hexadecimal representation of an unsigned C integer (lowercase).
+
+ * - ``X``
+ - Specified by the length modifier
+ - The hexadecimal representation of an unsigned C integer (uppercase).
+
+ * - ``c``
+ - :c:expr:`int`
+ - A single character.
+
+ * - ``s``
+ - :c:expr:`const char*` or :c:expr:`const wchar_t*`
+ - A null-terminated C character array.
+
+ * - ``p``
+ - :c:expr:`const void*`
+ - The hex representation of a C pointer.
+ Mostly equivalent to ``printf("%p")`` except that it is guaranteed to
+ start with the literal ``0x`` regardless of what the platform's
+ ``printf`` yields.
+
+ * - ``A``
+ - :c:expr:`PyObject*`
+ - The result of calling :func:`ascii`.
+
+ * - ``U``
+ - :c:expr:`PyObject*`
+ - A Unicode object.
+
+ * - ``V``
+ - :c:expr:`PyObject*`, :c:expr:`const char*` or :c:expr:`const wchar_t*`
+ - A Unicode object (which may be ``NULL``) and a null-terminated
+ C character array as a second parameter (which will be used,
+ if the first parameter is ``NULL``).
+
+ * - ``S``
+ - :c:expr:`PyObject*`
+ - The result of calling :c:func:`PyObject_Str`.
+
+ * - ``R``
+ - :c:expr:`PyObject*`
+ - The result of calling :c:func:`PyObject_Repr`.
.. note::
The width formatter unit is number of characters rather than bytes.
- The precision formatter unit is number of bytes for ``"%s"`` and
+ The precision formatter unit is number of bytes or :c:type:`wchar_t`
+ items (if the length modifier ``l`` is used) for ``"%s"`` and
``"%V"`` (if the ``PyObject*`` argument is ``NULL``), and a number of
characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"``
(if the ``PyObject*`` argument is not ``NULL``).
- .. [1] For integer specifiers (d, u, ld, li, lu, lld, lli, llu, zd, zi,
- zu, i, x): the 0-conversion flag has effect even when a precision is given.
+ .. note::
+ Unlike to C :c:func:`printf` the ``0`` flag has effect even when
+ a precision is given for integer conversions (``d``, ``i``, ``u``, ``o``,
+ ``x``, or ``X``).
.. versionchanged:: 3.2
Support for ``"%lld"`` and ``"%llu"`` added.
@@ -498,6 +542,13 @@ APIs:
``"%V"``, ``"%S"``, ``"%R"`` added.
.. versionchanged:: 3.12
+ Support for conversion specifiers ``o`` and ``X``.
+ Support for length modifiers ``j`` and ``t``.
+ Length modifiers are now applied to all integer conversions.
+ Length modifier ``l`` is now applied to conversion specifiers ``s`` and ``V``.
+ Support for variable width and precision ``*``.
+ Support for flag ``-``.
+
An unrecognized format character now sets a :exc:`SystemError`.
In previous versions it caused all the rest of the format string to be
copied as-is to the result string, and any extra arguments discarded.
@@ -509,6 +560,15 @@ APIs:
arguments.
+.. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj)
+
+ Copy an instance of a Unicode subtype to a new true Unicode object if
+ necessary. If *obj* is already a true Unicode object (not a subtype),
+ return a new :term:`strong reference` to the object.
+
+ Objects other than Unicode or its subtypes will cause a :exc:`TypeError`.
+
+
.. c:function:: PyObject* PyUnicode_FromEncodedObject(PyObject *obj, \
const char *encoding, const char *errors)
@@ -541,7 +601,7 @@ APIs:
Py_ssize_t how_many)
Copy characters from one Unicode object into another. This function performs
- character conversion when necessary and falls back to :c:func:`memcpy` if
+ character conversion when necessary and falls back to :c:func:`!memcpy` if
possible. Returns ``-1`` and sets an exception on error, otherwise returns
the number of copied characters.
@@ -616,15 +676,6 @@ APIs:
.. versionadded:: 3.3
-.. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj)
-
- Copy an instance of a Unicode subtype to a new true Unicode object if
- necessary. If *obj* is already a true Unicode object (not a subtype),
- return the reference with incremented refcount.
-
- Objects other than Unicode or its subtypes will cause a :exc:`TypeError`.
-
-
Locale Encoding
"""""""""""""""
@@ -663,7 +714,7 @@ system.
.. c:function:: PyObject* PyUnicode_DecodeLocale(const char *str, const char *errors)
Similar to :c:func:`PyUnicode_DecodeLocaleAndSize`, but compute the string
- length using :c:func:`strlen`.
+ length using :c:func:`!strlen`.
.. versionadded:: 3.3
@@ -788,11 +839,11 @@ conversion function:
wchar_t Support
"""""""""""""""
-:c:expr:`wchar_t` support for platforms which support it:
+:c:type:`wchar_t` support for platforms which support it:
.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *w, Py_ssize_t size)
- Create a Unicode object from the :c:expr:`wchar_t` buffer *w* of the given *size*.
+ Create a Unicode object from the :c:type:`wchar_t` buffer *w* of the given *size*.
Passing ``-1`` as the *size* indicates that the function must itself compute the length,
using wcslen.
Return ``NULL`` on failure.
@@ -800,9 +851,9 @@ wchar_t Support
.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size)
- Copy the Unicode object contents into the :c:expr:`wchar_t` buffer *w*. At most
- *size* :c:expr:`wchar_t` characters are copied (excluding a possibly trailing
- null termination character). Return the number of :c:expr:`wchar_t` characters
+ Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most
+ *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing
+ null termination character). Return the number of :c:type:`wchar_t` characters
copied or ``-1`` in case of an error. Note that the resulting :c:expr:`wchar_t*`
string may or may not be null-terminated. It is the responsibility of the caller
to make sure that the :c:expr:`wchar_t*` string is null-terminated in case this is
@@ -816,12 +867,12 @@ wchar_t Support
Convert the Unicode object to a wide character string. The output string
always ends with a null character. If *size* is not ``NULL``, write the number
of wide characters (excluding the trailing null termination character) into
- *\*size*. Note that the resulting :c:expr:`wchar_t` string might contain
+ *\*size*. Note that the resulting :c:type:`wchar_t` string might contain
null characters, which would cause the string to be truncated when used with
most C functions. If *size* is ``NULL`` and the :c:expr:`wchar_t*` string
contains null characters a :exc:`ValueError` is raised.
- Returns a buffer allocated by :c:func:`PyMem_New` (use
+ Returns a buffer allocated by :c:macro:`PyMem_New` (use
:c:func:`PyMem_Free` to free it) on success. On error, returns ``NULL``
and *\*size* is undefined. Raises a :exc:`MemoryError` if memory allocation
is failed.
@@ -920,8 +971,8 @@ These are the UTF-8 codec APIs:
returned buffer always has an extra null byte appended (not included in
*size*), regardless of whether there are any other null code points.
- In the case of an error, ``NULL`` is returned with an exception set and no
- *size* is stored.
+ On error, set an exception, set *size* to ``-1`` (if it's not NULL) and
+ return ``NULL``.
This caches the UTF-8 representation of the string in the Unicode object, and
subsequent calls will return a pointer to the same buffer. The caller is not
@@ -934,7 +985,7 @@ These are the UTF-8 codec APIs:
The return type is now ``const char *`` rather of ``char *``.
.. versionchanged:: 3.10
- This function is a part of the :ref:`limited API `.
+ This function is a part of the :ref:`limited API `.
.. c:function:: const char* PyUnicode_AsUTF8(PyObject *unicode)
@@ -1154,9 +1205,9 @@ Character Map Codecs
This codec is special in that it can be used to implement many different codecs
(and this is in fact what was done to obtain most of the standard codecs
-included in the :mod:`encodings` package). The codec uses mappings to encode and
+included in the :mod:`!encodings` package). The codec uses mappings to encode and
decode characters. The mapping objects provided must support the
-:meth:`__getitem__` mapping interface; dictionaries and sequences work well.
+:meth:`~object.__getitem__` mapping interface; dictionaries and sequences work well.
These are the mapping codec APIs:
@@ -1199,7 +1250,7 @@ The following codec API is special in that maps Unicode to Unicode.
The mapping table must map Unicode ordinal integers to Unicode ordinal integers
or ``None`` (causing deletion of the character).
- Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries
+ Mapping tables need only provide the :meth:`~object.__getitem__` interface; dictionaries
and sequences work well. Unmapped character ordinals (ones which cause a
:exc:`LookupError`) are left untouched and are copied as-is.
@@ -1241,7 +1292,7 @@ the user settings on the machine running the codec.
Encode the Unicode object using the specified code page and return a Python
bytes object. Return ``NULL`` if an exception was raised by the codec. Use
- :c:data:`CP_ACP` code page to get the MBCS encoder.
+ :c:macro:`!CP_ACP` code page to get the MBCS encoder.
.. versionadded:: 3.3
@@ -1345,6 +1396,28 @@ They all return ``NULL`` or ``-1`` if an exception occurs.
:c:func:`PyErr_Occurred` to check for errors.
+.. c:function:: int PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *string, Py_ssize_t size)
+
+ Compare a Unicode object with a char buffer which is interpreted as
+ being UTF-8 or ASCII encoded and return true (``1``) if they are equal,
+ or false (``0``) otherwise.
+ If the Unicode object contains surrogate characters or
+ the C string is not valid UTF-8, false (``0``) is returned.
+
+ This function does not raise exceptions.
+
+ .. versionadded:: 3.13
+
+
+.. c:function:: int PyUnicode_EqualToUTF8(PyObject *unicode, const char *string)
+
+ Similar to :c:func:`PyUnicode_EqualToUTF8AndSize`, but compute *string*
+ length using :c:func:`!strlen`.
+ If the Unicode object contains null characters, false (``0``) is returned.
+
+ .. versionadded:: 3.13
+
+
.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string)
Compare a Unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less
@@ -1360,11 +1433,11 @@ They all return ``NULL`` or ``-1`` if an exception occurs.
Rich compare two Unicode strings and return one of the following:
* ``NULL`` in case an exception was raised
- * :const:`Py_True` or :const:`Py_False` for successful comparisons
- * :const:`Py_NotImplemented` in case the type combination is unknown
+ * :c:data:`Py_True` or :c:data:`Py_False` for successful comparisons
+ * :c:data:`Py_NotImplemented` in case the type combination is unknown
- Possible values for *op* are :const:`Py_GT`, :const:`Py_GE`, :const:`Py_EQ`,
- :const:`Py_NE`, :const:`Py_LT`, and :const:`Py_LE`.
+ Possible values for *op* are :c:macro:`Py_GT`, :c:macro:`Py_GE`, :c:macro:`Py_EQ`,
+ :c:macro:`Py_NE`, :c:macro:`Py_LT`, and :c:macro:`Py_LE`.
.. c:function:: PyObject* PyUnicode_Format(PyObject *format, PyObject *args)
@@ -1387,11 +1460,11 @@ They all return ``NULL`` or ``-1`` if an exception occurs.
Intern the argument *\*string* in place. The argument must be the address of a
pointer variable pointing to a Python Unicode string object. If there is an
existing interned string that is the same as *\*string*, it sets *\*string* to
- it (decrementing the reference count of the old string object and incrementing
- the reference count of the interned string object), otherwise it leaves
- *\*string* alone and interns it (incrementing its reference count).
- (Clarification: even though there is a lot of talk about reference counts, think
- of this function as reference-count-neutral; you own the object after the call
+ it (releasing the reference to the old string object and creating a new
+ :term:`strong reference` to the interned string object), otherwise it leaves
+ *\*string* alone and interns it (creating a new :term:`strong reference`).
+ (Clarification: even though there is a lot of talk about references, think
+ of this function as reference-neutral; you own the object after the call
if and only if you owned it before the call.)
diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst
index a805b564763c407..48ae54acebe8879 100644
--- a/Doc/c-api/utilities.rst
+++ b/Doc/c-api/utilities.rst
@@ -17,5 +17,7 @@ and parsing function arguments and constructing Python values from C values.
marshal.rst
arg.rst
conversion.rst
+ hash.rst
reflection.rst
codec.rst
+ perfmaps.rst
diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst
index 513856d8a48d70f..324518c035096b4 100644
--- a/Doc/c-api/veryhigh.rst
+++ b/Doc/c-api/veryhigh.rst
@@ -12,12 +12,12 @@ file or a buffer, but they will not let you interact in a more detailed way with
the interpreter.
Several of these functions accept a start symbol from the grammar as a
-parameter. The available start symbols are :const:`Py_eval_input`,
-:const:`Py_file_input`, and :const:`Py_single_input`. These are described
+parameter. The available start symbols are :c:data:`Py_eval_input`,
+:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described
following the functions which accept them as parameters.
Note also that several of these functions take :c:expr:`FILE*` parameters. One
-particular issue which needs to be handled carefully is that the :c:expr:`FILE`
+particular issue which needs to be handled carefully is that the :c:type:`FILE`
structure for different C libraries can be different and incompatible. Under
Windows (at least), it is possible for dynamically linked extensions to actually
use different libraries, so care should be taken that :c:expr:`FILE*` parameters
@@ -167,6 +167,10 @@ the same library that the Python runtime is using.
event loops, as done in the :file:`Modules/_tkinter.c` in the
Python source code.
+ .. versionchanged:: 3.12
+ This function is only called from the
+ :ref:`main interpreter `.
+
.. c:var:: char* (*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *)
@@ -187,6 +191,10 @@ the same library that the Python runtime is using.
:c:func:`PyMem_RawRealloc`, instead of being allocated by
:c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`.
+ .. versionchanged:: 3.12
+ This function is only called from the
+ :ref:`main interpreter `.
+
.. c:function:: PyObject* PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals)
This is a simplified interface to :c:func:`PyRun_StringFlags` below, leaving
@@ -248,8 +256,8 @@ the same library that the Python runtime is using.
Parse and compile the Python source code in *str*, returning the resulting code
object. The start token is given by *start*; this can be used to constrain the
- code which can be compiled and should be :const:`Py_eval_input`,
- :const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by
+ code which can be compiled and should be :c:data:`Py_eval_input`,
+ :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by
*filename* is used to construct the code object and may appear in tracebacks or
:exc:`SyntaxError` exception messages. This returns ``NULL`` if the code
cannot be parsed or compiled.
@@ -345,7 +353,7 @@ the same library that the Python runtime is using.
executed, it is passed as ``PyCompilerFlags *flags``. In this case, ``from
__future__ import`` can modify *flags*.
- Whenever ``PyCompilerFlags *flags`` is ``NULL``, :attr:`cf_flags` is treated as
+ Whenever ``PyCompilerFlags *flags`` is ``NULL``, :c:member:`~PyCompilerFlags.cf_flags` is treated as
equal to ``0``, and any modification due to ``from __future__ import`` is
discarded.
@@ -359,7 +367,7 @@ the same library that the Python runtime is using.
initialized to ``PY_MINOR_VERSION``.
The field is ignored by default, it is used if and only if
- ``PyCF_ONLY_AST`` flag is set in *cf_flags*.
+ ``PyCF_ONLY_AST`` flag is set in :c:member:`~PyCompilerFlags.cf_flags`.
.. versionchanged:: 3.8
Added *cf_feature_version* field.
diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst
index ace743ba01c5f57..038f54a9751fd13 100644
--- a/Doc/c-api/weakref.rst
+++ b/Doc/c-api/weakref.rst
@@ -11,20 +11,20 @@ simple reference object, and the second acts as a proxy for the original object
as much as it can.
-.. c:function:: int PyWeakref_Check(ob)
+.. c:function:: int PyWeakref_Check(PyObject *ob)
- Return true if *ob* is either a reference or proxy object. This function
+ Return non-zero if *ob* is either a reference or proxy object. This function
always succeeds.
-.. c:function:: int PyWeakref_CheckRef(ob)
+.. c:function:: int PyWeakref_CheckRef(PyObject *ob)
- Return true if *ob* is a reference object. This function always succeeds.
+ Return non-zero if *ob* is a reference object. This function always succeeds.
-.. c:function:: int PyWeakref_CheckProxy(ob)
+.. c:function:: int PyWeakref_CheckProxy(PyObject *ob)
- Return true if *ob* is a proxy object. This function always succeeds.
+ Return non-zero if *ob* is a proxy object. This function always succeeds.
.. c:function:: PyObject* PyWeakref_NewRef(PyObject *ob, PyObject *callback)
@@ -51,10 +51,23 @@ as much as it can.
``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`.
+.. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj)
+
+ Get a :term:`strong reference` to the referenced object from a weak
+ reference, *ref*, into *\*pobj*.
+
+ * On success, set *\*pobj* to a new :term:`strong reference` to the
+ referenced object and return 1.
+ * If the reference is dead, set *\*pobj* to ``NULL`` and return 0.
+ * On error, raise an exception and return -1.
+
+ .. versionadded:: 3.13
+
+
.. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref)
- Return the referenced object from a weak reference, *ref*. If the referent is
- no longer live, returns :const:`Py_None`.
+ Return a :term:`borrowed reference` to the referenced object from a weak
+ reference, *ref*. If the referent is no longer live, returns ``Py_None``.
.. note::
@@ -63,7 +76,23 @@ as much as it can.
except when it cannot be destroyed before the last usage of the borrowed
reference.
+ .. deprecated-removed:: 3.13 3.15
+ Use :c:func:`PyWeakref_GetRef` instead.
+
.. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref)
Similar to :c:func:`PyWeakref_GetObject`, but does no error checking.
+
+ .. deprecated-removed:: 3.13 3.15
+ Use :c:func:`PyWeakref_GetRef` instead.
+
+
+.. c:function:: void PyObject_ClearWeakRefs(PyObject *object)
+
+ This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler
+ to clear weak references.
+
+ This iterates through the weak references for *object* and calls callbacks
+ for those references which have one. It returns when all callbacks have
+ been attempted.
diff --git a/Doc/conf.py b/Doc/conf.py
index be1c9fff51a277b..f1b411126c4e871 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -13,9 +13,25 @@
# General configuration
# ---------------------
-extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest',
- 'pyspecific', 'c_annotations', 'escape4chm',
- 'asdl_highlight', 'peg_highlight', 'glossary_search']
+extensions = [
+ 'asdl_highlight',
+ 'c_annotations',
+ 'escape4chm',
+ 'glossary_search',
+ 'peg_highlight',
+ 'pyspecific',
+ 'sphinx.ext.coverage',
+ 'sphinx.ext.doctest',
+]
+
+# Skip if downstream redistributors haven't installed it
+try:
+ import sphinxext.opengraph
+except ImportError:
+ pass
+else:
+ extensions.append('sphinxext.opengraph')
+
doctest_global_setup = '''
try:
@@ -50,14 +66,183 @@
highlight_language = 'python3'
# Minimum version of sphinx required
-needs_sphinx = '3.2'
+needs_sphinx = '4.2'
+# Ignore any .rst files in the includes/ directory;
+# they're embedded in pages but not rendered individually.
# Ignore any .rst files in the venv/ directory.
-exclude_patterns = ['venv/*', 'README.rst']
+exclude_patterns = ['includes/*.rst', 'venv/*', 'README.rst']
venvdir = os.getenv('VENVDIR')
if venvdir is not None:
exclude_patterns.append(venvdir + '/*')
+nitpick_ignore = [
+ # Standard C functions
+ ('c:func', 'calloc'),
+ ('c:func', 'dlopen'),
+ ('c:func', 'exec'),
+ ('c:func', 'fcntl'),
+ ('c:func', 'fork'),
+ ('c:func', 'free'),
+ ('c:func', 'gmtime'),
+ ('c:func', 'localtime'),
+ ('c:func', 'main'),
+ ('c:func', 'malloc'),
+ ('c:func', 'printf'),
+ ('c:func', 'realloc'),
+ ('c:func', 'snprintf'),
+ ('c:func', 'sprintf'),
+ ('c:func', 'stat'),
+ ('c:func', 'system'),
+ ('c:func', 'time'),
+ ('c:func', 'vsnprintf'),
+ # Standard C types
+ ('c:type', 'FILE'),
+ ('c:type', 'int64_t'),
+ ('c:type', 'intmax_t'),
+ ('c:type', 'off_t'),
+ ('c:type', 'ptrdiff_t'),
+ ('c:type', 'siginfo_t'),
+ ('c:type', 'size_t'),
+ ('c:type', 'ssize_t'),
+ ('c:type', 'time_t'),
+ ('c:type', 'uint64_t'),
+ ('c:type', 'uintmax_t'),
+ ('c:type', 'uintptr_t'),
+ ('c:type', 'va_list'),
+ ('c:type', 'wchar_t'),
+ ('c:type', '__int64'),
+ ('c:type', 'unsigned __int64'),
+ # Standard C structures
+ ('c:struct', 'in6_addr'),
+ ('c:struct', 'in_addr'),
+ ('c:struct', 'stat'),
+ ('c:struct', 'statvfs'),
+ # Standard C macros
+ ('c:macro', 'LLONG_MAX'),
+ ('c:macro', 'LLONG_MIN'),
+ ('c:macro', 'LONG_MAX'),
+ ('c:macro', 'LONG_MIN'),
+ # Standard C variables
+ ('c:data', 'errno'),
+ # Standard environment variables
+ ('envvar', 'BROWSER'),
+ ('envvar', 'COLUMNS'),
+ ('envvar', 'COMSPEC'),
+ ('envvar', 'DISPLAY'),
+ ('envvar', 'HOME'),
+ ('envvar', 'HOMEDRIVE'),
+ ('envvar', 'HOMEPATH'),
+ ('envvar', 'IDLESTARTUP'),
+ ('envvar', 'LANG'),
+ ('envvar', 'LANGUAGE'),
+ ('envvar', 'LC_ALL'),
+ ('envvar', 'LC_CTYPE'),
+ ('envvar', 'LC_COLLATE'),
+ ('envvar', 'LC_MESSAGES'),
+ ('envvar', 'LC_MONETARY'),
+ ('envvar', 'LC_NUMERIC'),
+ ('envvar', 'LC_TIME'),
+ ('envvar', 'LINES'),
+ ('envvar', 'LOGNAME'),
+ ('envvar', 'PAGER'),
+ ('envvar', 'PATH'),
+ ('envvar', 'PATHEXT'),
+ ('envvar', 'SOURCE_DATE_EPOCH'),
+ ('envvar', 'TEMP'),
+ ('envvar', 'TERM'),
+ ('envvar', 'TMP'),
+ ('envvar', 'TMPDIR'),
+ ('envvar', 'TZ'),
+ ('envvar', 'USER'),
+ ('envvar', 'USERNAME'),
+ ('envvar', 'USERPROFILE'),
+]
+
+# Temporary undocumented names.
+# In future this list must be empty.
+nitpick_ignore += [
+ # C API: Standard Python exception classes
+ ('c:data', 'PyExc_ArithmeticError'),
+ ('c:data', 'PyExc_AssertionError'),
+ ('c:data', 'PyExc_AttributeError'),
+ ('c:data', 'PyExc_BaseException'),
+ ('c:data', 'PyExc_BlockingIOError'),
+ ('c:data', 'PyExc_BrokenPipeError'),
+ ('c:data', 'PyExc_BufferError'),
+ ('c:data', 'PyExc_ChildProcessError'),
+ ('c:data', 'PyExc_ConnectionAbortedError'),
+ ('c:data', 'PyExc_ConnectionError'),
+ ('c:data', 'PyExc_ConnectionRefusedError'),
+ ('c:data', 'PyExc_ConnectionResetError'),
+ ('c:data', 'PyExc_EOFError'),
+ ('c:data', 'PyExc_Exception'),
+ ('c:data', 'PyExc_FileExistsError'),
+ ('c:data', 'PyExc_FileNotFoundError'),
+ ('c:data', 'PyExc_FloatingPointError'),
+ ('c:data', 'PyExc_GeneratorExit'),
+ ('c:data', 'PyExc_ImportError'),
+ ('c:data', 'PyExc_IndentationError'),
+ ('c:data', 'PyExc_IndexError'),
+ ('c:data', 'PyExc_InterruptedError'),
+ ('c:data', 'PyExc_IsADirectoryError'),
+ ('c:data', 'PyExc_KeyboardInterrupt'),
+ ('c:data', 'PyExc_KeyError'),
+ ('c:data', 'PyExc_LookupError'),
+ ('c:data', 'PyExc_MemoryError'),
+ ('c:data', 'PyExc_ModuleNotFoundError'),
+ ('c:data', 'PyExc_NameError'),
+ ('c:data', 'PyExc_NotADirectoryError'),
+ ('c:data', 'PyExc_NotImplementedError'),
+ ('c:data', 'PyExc_OSError'),
+ ('c:data', 'PyExc_OverflowError'),
+ ('c:data', 'PyExc_PermissionError'),
+ ('c:data', 'PyExc_ProcessLookupError'),
+ ('c:data', 'PyExc_RecursionError'),
+ ('c:data', 'PyExc_ReferenceError'),
+ ('c:data', 'PyExc_RuntimeError'),
+ ('c:data', 'PyExc_StopAsyncIteration'),
+ ('c:data', 'PyExc_StopIteration'),
+ ('c:data', 'PyExc_SyntaxError'),
+ ('c:data', 'PyExc_SystemError'),
+ ('c:data', 'PyExc_SystemExit'),
+ ('c:data', 'PyExc_TabError'),
+ ('c:data', 'PyExc_TimeoutError'),
+ ('c:data', 'PyExc_TypeError'),
+ ('c:data', 'PyExc_UnboundLocalError'),
+ ('c:data', 'PyExc_UnicodeDecodeError'),
+ ('c:data', 'PyExc_UnicodeEncodeError'),
+ ('c:data', 'PyExc_UnicodeError'),
+ ('c:data', 'PyExc_UnicodeTranslateError'),
+ ('c:data', 'PyExc_ValueError'),
+ ('c:data', 'PyExc_ZeroDivisionError'),
+ # C API: Standard Python warning classes
+ ('c:data', 'PyExc_BytesWarning'),
+ ('c:data', 'PyExc_DeprecationWarning'),
+ ('c:data', 'PyExc_FutureWarning'),
+ ('c:data', 'PyExc_ImportWarning'),
+ ('c:data', 'PyExc_PendingDeprecationWarning'),
+ ('c:data', 'PyExc_ResourceWarning'),
+ ('c:data', 'PyExc_RuntimeWarning'),
+ ('c:data', 'PyExc_SyntaxWarning'),
+ ('c:data', 'PyExc_UnicodeWarning'),
+ ('c:data', 'PyExc_UserWarning'),
+ ('c:data', 'PyExc_Warning'),
+ # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot
+ # be resolved, as the method is currently undocumented. For context, see
+ # https://github.com/python/cpython/pull/103289.
+ ('py:meth', '_SubParsersAction.add_parser'),
+]
+
+# gh-106948: Copy standard C types declared in the "c:type" domain to the
+# "c:identifier" domain, since "c:function" markup looks for types in the
+# "c:identifier" domain. Use list() to not iterate on items which are being
+# added
+for role, name in list(nitpick_ignore):
+ if role == 'c:type':
+ nitpick_ignore.append(('c:identifier', name))
+del role, name
+
# Disable Docutils smartquotes for several translations
smartquotes_excludes = {
'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'],
@@ -66,6 +251,11 @@
# Avoid a warning with Sphinx >= 2.0
master_doc = 'contents'
+# Allow translation of index directives
+gettext_additional_targets = [
+ 'index',
+]
+
# Options for HTML output
# -----------------------
@@ -89,9 +279,17 @@
# Short title used e.g. for HTML tags.
html_short_title = '%s Documentation' % release
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-html_last_updated_fmt = '%b %d, %Y'
+# Deployment preview information
+# (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html)
+repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL")
+html_context = {
+ "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external",
+ "repository_url": repository_url.removesuffix(".git") if repository_url else None,
+ "pr_id": os.getenv("READTHEDOCS_VERSION")
+}
+
+# This 'Last updated on:' timestamp is inserted at the bottom of every page.
+html_last_updated_fmt = time.strftime('%b %d, %Y (%H:%M UTC)', time.gmtime())
# Path to find HTML templates.
templates_path = ['tools/templates']
@@ -114,7 +312,7 @@
html_use_opensearch = 'https://docs.python.org/' + version
# Additional static files.
-html_static_path = ['tools/static']
+html_static_path = ['_static', 'tools/static']
# Output file base name for HTML help builder.
htmlhelp_basename = 'python' + release.replace('.', '')
@@ -155,8 +353,6 @@
latex_documents = [
('c-api/index', 'c-api.tex',
'The Python/C API', _stdauthor, 'manual'),
- ('distributing/index', 'distributing.tex',
- 'Distributing Python Modules', _stdauthor, 'manual'),
('extending/index', 'extending.tex',
'Extending and Embedding Python', _stdauthor, 'manual'),
('installing/index', 'installing.tex',
@@ -196,8 +392,6 @@
# match any of the following regexes (using re.match).
coverage_ignore_modules = [
r'[T|t][k|K]',
- r'Tix',
- r'distutils.*',
]
coverage_ignore_functions = [
@@ -229,8 +423,49 @@
# Options for the link checker
# ----------------------------
-# Ignore certain URLs.
-linkcheck_ignore = [r'https://bugs.python.org/(issue)?\d+']
+linkcheck_allowed_redirects = {
+ # bpo-NNNN -> BPO -> GH Issues
+ r'https://bugs.python.org/issue\?@action=redirect&bpo=\d+': r'https://github.com/python/cpython/issues/\d+',
+ # GH-NNNN used to refer to pull requests
+ r'https://github.com/python/cpython/issues/\d+': r'https://github.com/python/cpython/pull/\d+',
+ # :source:`something` linking files in the repository
+ r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*',
+ # Intentional HTTP use at Misc/NEWS.d/3.5.0a1.rst
+ r'http://www.python.org/$': 'https://www.python.org/$',
+ # Used in license page, keep as is
+ r'https://www.zope.org/': r'https://www.zope.dev/',
+ # Microsoft's redirects to learn.microsoft.com
+ r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*',
+ r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*',
+ r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*',
+ # Language redirects
+ r'https://toml.io': 'https://toml.io/en/',
+ r'https://www.redhat.com': 'https://www.redhat.com/en',
+ # Other redirects
+ r'https://www.boost.org/libs/.+': r'https://www.boost.org/doc/libs/\d_\d+_\d/.+',
+ r'https://support.microsoft.com/en-us/help/\d+': 'https://support.microsoft.com/en-us/topic/.+',
+ r'https://perf.wiki.kernel.org$': 'https://perf.wiki.kernel.org/index.php/Main_Page',
+ r'https://www.sqlite.org': 'https://www.sqlite.org/index.html',
+ r'https://mitpress.mit.edu/sicp$': 'https://mitpress.mit.edu/9780262510875/structure-and-interpretation-of-computer-programs/',
+ r'https://www.python.org/psf/': 'https://www.python.org/psf-landing/',
+}
+
+linkcheck_anchors_ignore = [
+ # ignore anchors that start with a '/', e.g. Wikipedia media files:
+ # https://en.wikipedia.org/wiki/Walrus#/media/File:Pacific_Walrus_-_Bull_(8247646168).jpg
+ r'\/.*',
+]
+
+linkcheck_ignore = [
+ # The crawler gets "Anchor not found"
+ r'https://developer.apple.com/documentation/.+?#.*',
+ r'https://devguide.python.org.+?/#.*',
+ r'https://github.com.+?#.*',
+ # Robot crawlers not allowed: "403 Client Error: Forbidden"
+ r'https://support.enthought.com/hc/.*',
+ # SSLError CertificateError, even though it is valid
+ r'https://unix.org/version2/whatsnew/lp64_wp.html',
+]
# Options for extensions
@@ -239,3 +474,13 @@
# Relative filename of the data files
refcount_file = 'data/refcounts.dat'
stable_abi_file = 'data/stable_abi.dat'
+
+# sphinxext-opengraph config
+ogp_site_url = 'https://docs.python.org/3/'
+ogp_site_name = 'Python documentation'
+ogp_image = '_static/og-image.png'
+ogp_custom_meta_tags = [
+ '',
+ '',
+ '',
+]
diff --git a/Doc/constraints.txt b/Doc/constraints.txt
new file mode 100644
index 000000000000000..147de1271eb2b7f
--- /dev/null
+++ b/Doc/constraints.txt
@@ -0,0 +1,24 @@
+# We have upper bounds on our transitive dependencies here
+# To avoid new releases unexpectedly breaking our build.
+# This file can be updated on an ad-hoc basis,
+# though it will probably have to be updated
+# whenever Doc/requirements.txt is updated.
+
+# Direct dependencies of Sphinx
+babel<3
+colorama<0.5
+imagesize<1.5
+Jinja2<3.2
+packaging<24
+Pygments>=2.16.1,<3
+requests<3
+snowballstemmer<3
+sphinxcontrib-applehelp<1.1
+sphinxcontrib-devhelp<1.1
+sphinxcontrib-htmlhelp<2.1
+sphinxcontrib-jsmath<1.1
+sphinxcontrib-qthelp<1.1
+sphinxcontrib-serializinghtml<1.2
+
+# Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above)
+MarkupSafe<2.2
diff --git a/Doc/contents.rst b/Doc/contents.rst
index 464f93bdf85f95b..24ceacb0076b5eb 100644
--- a/Doc/contents.rst
+++ b/Doc/contents.rst
@@ -11,7 +11,6 @@
library/index.rst
extending/index.rst
c-api/index.rst
- distributing/index.rst
installing/index.rst
howto/index.rst
faq/index.rst
@@ -21,10 +20,3 @@
bugs.rst
copyright.rst
license.rst
-
-.. to include legacy packaging docs in build
-
-.. toctree::
- :hidden:
-
- install/index.rst
diff --git a/Doc/copyright.rst b/Doc/copyright.rst
index e64a49328b47232..9b71683155eebe7 100644
--- a/Doc/copyright.rst
+++ b/Doc/copyright.rst
@@ -4,7 +4,7 @@ Copyright
Python and this documentation is:
-Copyright © 2001-2022 Python Software Foundation. All rights reserved.
+Copyright © 2001-2023 Python Software Foundation. All rights reserved.
Copyright © 2000 BeOpen.com. All rights reserved.
diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
index 51ccacf13f9e3be..ef9ac1617a284be 100644
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -606,6 +606,9 @@ PyErr_GetExcInfo:PyObject**:ptype:+1:
PyErr_GetExcInfo:PyObject**:pvalue:+1:
PyErr_GetExcInfo:PyObject**:ptraceback:+1:
+PyErr_GetRaisedException:PyObject*::+1:
+PyErr_SetRaisedException::::
+
PyErr_GivenExceptionMatches:int:::
PyErr_GivenExceptionMatches:PyObject*:given:0:
PyErr_GivenExceptionMatches:PyObject*:exc:0:
@@ -761,8 +764,6 @@ PyErr_WarnFormat::...::
PyErr_WriteUnraisable:void:::
PyErr_WriteUnraisable:PyObject*:obj:0:
-PyEval_AcquireLock:void:::
-
PyEval_AcquireThread:void:::
PyEval_AcquireThread:PyThreadState*:tstate::
@@ -780,10 +781,6 @@ PyEval_GetFuncDesc:PyObject*:func:0:
PyEval_GetFuncName:const char*:::
PyEval_GetFuncName:PyObject*:func:0:
-PyEval_InitThreads:void:::
-
-PyEval_ReleaseLock:void:::
-
PyEval_ReleaseThread:void:::
PyEval_ReleaseThread:PyThreadState*:tstate::
@@ -836,6 +833,8 @@ PyEval_EvalFrameEx:int:throwflag::
PyEval_MergeCompilerFlags:int:::
PyEval_MergeCompilerFlags:PyCompilerFlags*:cf::
+PyException_GetArgs:PyObject*::+1:
+
PyException_GetCause:PyObject*::+1:
PyException_GetCause:PyObject*:ex:0:
@@ -975,6 +974,9 @@ PyCoro_New:PyFrameObject*:frame:0:
PyCoro_New:PyObject*:name:0:
PyCoro_New:PyObject*:qualname:0:
+PyImport_AddModuleRef:PyObject*::+1:
+PyImport_AddModuleRef:const char*:name::
+
PyImport_AddModule:PyObject*::0:reference borrowed from sys.modules
PyImport_AddModule:const char*:name::
@@ -1018,10 +1020,10 @@ PyImport_Import:PyObject*::+1:
PyImport_Import:PyObject*:name:0:
PyImport_ImportFrozenModule:int:::
-PyImport_ImportFrozenModule:const char*:::
+PyImport_ImportFrozenModule:const char*:name::
PyImport_ImportFrozenModuleObject:int:::
-PyImport_ImportFrozenModuleObject:PyObject*::+1:
+PyImport_ImportFrozenModuleObject:PyObject*:name:+1:
PyImport_ImportModule:PyObject*::+1:
PyImport_ImportModule:const char*:name::
@@ -1579,21 +1581,6 @@ PyOS_FSPath:PyObject*:path:0:
PyObject_ASCII:PyObject*::+1:
PyObject_ASCII:PyObject*:o:0:
-PyObject_AsCharBuffer:int:::
-PyObject_AsCharBuffer:PyObject*:obj:0:
-PyObject_AsCharBuffer:const char**:buffer::
-PyObject_AsCharBuffer:Py_ssize_t*:buffer_len::
-
-PyObject_AsReadBuffer:int:::
-PyObject_AsReadBuffer:PyObject*:obj:0:
-PyObject_AsReadBuffer:const void**:buffer::
-PyObject_AsReadBuffer:Py_ssize_t*:buffer_len::
-
-PyObject_AsWriteBuffer:int:::
-PyObject_AsWriteBuffer:PyObject*:obj:0:
-PyObject_AsWriteBuffer:void**:buffer::
-PyObject_AsWriteBuffer:Py_ssize_t*:buffer_len::
-
PyObject_Bytes:PyObject*::+1:
PyObject_Bytes:PyObject*:o:0:
@@ -1810,6 +1797,9 @@ PyObject_Size:PyObject*:o:0:
PyObject_Str:PyObject*::+1:
PyObject_Str:PyObject*:o:0:
+Py_TYPE:PyObject*::0:
+Py_TYPE:PyObject*:ob:0:
+
PyObject_Type:PyObject*::+1:
PyObject_Type:PyObject*:o:0:
@@ -2369,76 +2359,56 @@ PyUnicode_KIND:PyObject*:o:0:
PyUnicode_MAX_CHAR_VALUE::::
PyUnicode_MAX_CHAR_VALUE:PyObject*:o:0:
-PyUnicode_AS_UNICODE:Py_UNICODE*:::
-PyUnicode_AS_UNICODE:PyObject*:o:0:
-
-PyUnicode_AS_DATA:const char*:::
-PyUnicode_AS_DATA:PyObject*:o:0:
-
Py_UNICODE_ISALNUM:int:::
-Py_UNICODE_ISALNUM:Py_UNICODE:ch::
+Py_UNICODE_ISALNUM:Py_UCS4:ch::
Py_UNICODE_ISALPHA:int:::
-Py_UNICODE_ISALPHA:Py_UNICODE:ch::
+Py_UNICODE_ISALPHA:Py_UCS4:ch::
Py_UNICODE_ISSPACE:int:::
-Py_UNICODE_ISSPACE:Py_UNICODE:ch::
+Py_UNICODE_ISSPACE:Py_UCS4:ch::
Py_UNICODE_ISLOWER:int:::
-Py_UNICODE_ISLOWER:Py_UNICODE:ch::
+Py_UNICODE_ISLOWER:Py_UCS4:ch::
Py_UNICODE_ISUPPER:int:::
-Py_UNICODE_ISUPPER:Py_UNICODE:ch::
+Py_UNICODE_ISUPPER:Py_UCS4:ch::
Py_UNICODE_ISTITLE:int:::
-Py_UNICODE_ISTITLE:Py_UNICODE:ch::
+Py_UNICODE_ISTITLE:Py_UCS4:ch::
Py_UNICODE_ISLINEBREAK:int:::
-Py_UNICODE_ISLINEBREAK:Py_UNICODE:ch::
+Py_UNICODE_ISLINEBREAK:Py_UCS4:ch::
Py_UNICODE_ISDECIMAL:int:::
-Py_UNICODE_ISDECIMAL:Py_UNICODE:ch::
+Py_UNICODE_ISDECIMAL:Py_UCS4:ch::
Py_UNICODE_ISDIGIT:int:::
-Py_UNICODE_ISDIGIT:Py_UNICODE:ch::
+Py_UNICODE_ISDIGIT:Py_UCS4:ch::
Py_UNICODE_ISNUMERIC:int:::
-Py_UNICODE_ISNUMERIC:Py_UNICODE:ch::
+Py_UNICODE_ISNUMERIC:Py_UCS4:ch::
Py_UNICODE_ISPRINTABLE:int:::
-Py_UNICODE_ISPRINTABLE:Py_UNICODE:ch::
+Py_UNICODE_ISPRINTABLE:Py_UCS4:ch::
-Py_UNICODE_TOLOWER:Py_UNICODE:::
-Py_UNICODE_TOLOWER:Py_UNICODE:ch::
+Py_UNICODE_TOLOWER:Py_UCS4:::
+Py_UNICODE_TOLOWER:Py_UCS4:ch::
-Py_UNICODE_TOUPPER:Py_UNICODE:::
-Py_UNICODE_TOUPPER:Py_UNICODE:ch::
+Py_UNICODE_TOUPPER:Py_UCS4:::
+Py_UNICODE_TOUPPER:Py_UCS4:ch::
-Py_UNICODE_TOTITLE:Py_UNICODE:::
-Py_UNICODE_TOTITLE:Py_UNICODE:ch::
+Py_UNICODE_TOTITLE:Py_UCS4:::
+Py_UNICODE_TOTITLE:Py_UCS4:ch::
Py_UNICODE_TODECIMAL:int:::
-Py_UNICODE_TODECIMAL:Py_UNICODE:ch::
+Py_UNICODE_TODECIMAL:Py_UCS4:ch::
Py_UNICODE_TODIGIT:int:::
-Py_UNICODE_TODIGIT:Py_UNICODE:ch::
+Py_UNICODE_TODIGIT:Py_UCS4:ch::
Py_UNICODE_TONUMERIC:double:::
-Py_UNICODE_TONUMERIC:Py_UNICODE:ch::
-
-PyUnicode_FromUnicode:PyObject*::+1:
-PyUnicode_FromUnicode:const Py_UNICODE*:u::
-PyUnicode_FromUnicode:Py_ssize_t:size::
-
-PyUnicode_AsUnicode:Py_UNICODE*:::
-PyUnicode_AsUnicode:PyObject*:unicode:0:
-
-PyUnicode_AsUnicodeAndSize:Py_UNICODE*:::
-PyUnicode_AsUnicodeAndSize:PyObject*:unicode:0:
-PyUnicode_AsUnicodeAndSize:Py_ssize_t*:size::
-
-PyUnicode_GetSize:Py_ssize_t:::
-PyUnicode_GetSize:PyObject*:unicode:0:
+Py_UNICODE_TONUMERIC:Py_UCS4:ch::
PyUnicode_FromObject:PyObject*::+1:
PyUnicode_FromObject:PyObject*:obj:0:
@@ -2843,6 +2813,10 @@ PyWeakref_GET_OBJECT:PyObject*:ref:0:
PyWeakref_GetObject:PyObject*::0:
PyWeakref_GetObject:PyObject*:ref:0:
+PyWeakref_GetRef:int:::
+PyWeakref_GetRef:PyObject*:ref:0:
+PyWeakref_GetRef:PyObject**:pobj:+1:
+
PyWeakref_NewProxy:PyObject*::+1:
PyWeakref_NewProxy:PyObject*:ob:0:
PyWeakref_NewProxy:PyObject*:callback:0:
diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat
index db8fc15d93d15a0..811b1bd84d24174 100644
--- a/Doc/data/stable_abi.dat
+++ b/Doc/data/stable_abi.dat
@@ -43,7 +43,6 @@ function,PyBytes_Size,3.2,,
var,PyBytes_Type,3.2,,
type,PyCFunction,3.2,,
type,PyCFunctionWithKeywords,3.2,,
-function,PyCFunction_Call,3.2,,
function,PyCFunction_GetFlags,3.2,,
function,PyCFunction_GetFunction,3.2,,
function,PyCFunction_GetSelf,3.2,,
@@ -112,7 +111,9 @@ function,PyDict_Copy,3.2,,
function,PyDict_DelItem,3.2,,
function,PyDict_DelItemString,3.2,,
function,PyDict_GetItem,3.2,,
+function,PyDict_GetItemRef,3.13,,
function,PyDict_GetItemString,3.2,,
+function,PyDict_GetItemStringRef,3.13,,
function,PyDict_GetItemWithError,3.2,,
function,PyDict_Items,3.2,,
function,PyDict_Keys,3.2,,
@@ -133,12 +134,14 @@ function,PyErr_BadInternalCall,3.2,,
function,PyErr_CheckSignals,3.2,,
function,PyErr_Clear,3.2,,
function,PyErr_Display,3.2,,
+function,PyErr_DisplayException,3.12,,
function,PyErr_ExceptionMatches,3.2,,
function,PyErr_Fetch,3.2,,
function,PyErr_Format,3.2,,
function,PyErr_FormatV,3.5,,
function,PyErr_GetExcInfo,3.7,,
function,PyErr_GetHandledException,3.11,,
+function,PyErr_GetRaisedException,3.12,,
function,PyErr_GivenExceptionMatches,3.2,,
function,PyErr_NewException,3.2,,
function,PyErr_NewExceptionWithDoc,3.2,,
@@ -168,6 +171,7 @@ function,PyErr_SetInterrupt,3.2,,
function,PyErr_SetInterruptEx,3.10,,
function,PyErr_SetNone,3.2,,
function,PyErr_SetObject,3.2,,
+function,PyErr_SetRaisedException,3.12,,
function,PyErr_SetString,3.2,,
function,PyErr_SyntaxLocation,3.2,,
function,PyErr_SyntaxLocationEx,3.7,,
@@ -175,11 +179,7 @@ function,PyErr_WarnEx,3.2,,
function,PyErr_WarnExplicit,3.2,,
function,PyErr_WarnFormat,3.2,,
function,PyErr_WriteUnraisable,3.2,,
-function,PyEval_AcquireLock,3.2,,
function,PyEval_AcquireThread,3.2,,
-function,PyEval_CallFunction,3.2,,
-function,PyEval_CallMethod,3.2,,
-function,PyEval_CallObjectWithKeywords,3.2,,
function,PyEval_EvalCode,3.2,,
function,PyEval_EvalCodeEx,3.2,,
function,PyEval_EvalFrame,3.2,,
@@ -190,12 +190,9 @@ function,PyEval_GetFuncDesc,3.2,,
function,PyEval_GetFuncName,3.2,,
function,PyEval_GetGlobals,3.2,,
function,PyEval_GetLocals,3.2,,
-function,PyEval_InitThreads,3.2,,
-function,PyEval_ReleaseLock,3.2,,
function,PyEval_ReleaseThread,3.2,,
function,PyEval_RestoreThread,3.2,,
function,PyEval_SaveThread,3.2,,
-function,PyEval_ThreadsInitialized,3.2,,
var,PyExc_ArithmeticError,3.2,,
var,PyExc_AssertionError,3.2,,
var,PyExc_AttributeError,3.2,,
@@ -266,9 +263,11 @@ var,PyExc_Warning,3.2,,
var,PyExc_WindowsError,3.7,on Windows,
var,PyExc_ZeroDivisionError,3.2,,
function,PyExceptionClass_Name,3.8,,
+function,PyException_GetArgs,3.12,,
function,PyException_GetCause,3.2,,
function,PyException_GetContext,3.2,,
function,PyException_GetTraceback,3.2,,
+function,PyException_SetArgs,3.12,,
function,PyException_SetCause,3.2,,
function,PyException_SetContext,3.2,,
function,PyException_SetTraceback,3.2,,
@@ -301,6 +300,7 @@ type,PyGetSetDef,3.2,,full-abi
var,PyGetSetDescr_Type,3.2,,
function,PyImport_AddModule,3.2,,
function,PyImport_AddModuleObject,3.7,,
+function,PyImport_AddModuleRef,3.13,,
function,PyImport_AppendInittab,3.2,,
function,PyImport_ExecCodeModule,3.2,,
function,PyImport_ExecCodeModuleEx,3.2,,
@@ -347,6 +347,7 @@ var,PyList_Type,3.2,,
type,PyLongObject,3.2,,opaque
var,PyLongRangeIter_Type,3.2,,
function,PyLong_AsDouble,3.2,,
+function,PyLong_AsInt,3.13,,
function,PyLong_AsLong,3.2,,
function,PyLong_AsLongAndOverflow,3.2,,
function,PyLong_AsLongLong,3.2,,
@@ -372,8 +373,12 @@ var,PyLong_Type,3.2,,
var,PyMap_Type,3.2,,
function,PyMapping_Check,3.2,,
function,PyMapping_GetItemString,3.2,,
+function,PyMapping_GetOptionalItem,3.13,,
+function,PyMapping_GetOptionalItemString,3.13,,
function,PyMapping_HasKey,3.2,,
function,PyMapping_HasKeyString,3.2,,
+function,PyMapping_HasKeyStringWithError,3.13,,
+function,PyMapping_HasKeyWithError,3.13,,
function,PyMapping_Items,3.2,,
function,PyMapping_Keys,3.2,,
function,PyMapping_Length,3.2,,
@@ -383,9 +388,15 @@ function,PyMapping_Values,3.2,,
function,PyMem_Calloc,3.7,,
function,PyMem_Free,3.2,,
function,PyMem_Malloc,3.2,,
+function,PyMem_RawCalloc,3.13,,
+function,PyMem_RawFree,3.13,,
+function,PyMem_RawMalloc,3.13,,
+function,PyMem_RawRealloc,3.13,,
function,PyMem_Realloc,3.2,,
type,PyMemberDef,3.2,,full-abi
var,PyMemberDescr_Type,3.2,,
+function,PyMember_GetOne,3.2,,
+function,PyMember_SetOne,3.2,,
function,PyMemoryView_FromBuffer,3.11,,
function,PyMemoryView_FromMemory,3.7,,
function,PyMemoryView_FromObject,3.2,,
@@ -397,6 +408,7 @@ type,PyModuleDef,3.2,,full-abi
type,PyModuleDef_Base,3.2,,full-abi
function,PyModuleDef_Init,3.5,,
var,PyModuleDef_Type,3.5,,
+function,PyModule_Add,3.13,,
function,PyModule_AddFunctions,3.7,,
function,PyModule_AddIntConstant,3.2,,
function,PyModule_AddObject,3.2,,
@@ -477,10 +489,7 @@ type,PyObject,3.2,,members
member,PyObject.ob_refcnt,3.2,,
member,PyObject.ob_type,3.2,,
function,PyObject_ASCII,3.2,,
-function,PyObject_AsCharBuffer,3.2,,
function,PyObject_AsFileDescriptor,3.2,,
-function,PyObject_AsReadBuffer,3.2,,
-function,PyObject_AsWriteBuffer,3.2,,
function,PyObject_Bytes,3.2,,
function,PyObject_Call,3.2,,
function,PyObject_CallFunction,3.2,,
@@ -491,9 +500,10 @@ function,PyObject_CallNoArgs,3.10,,
function,PyObject_CallObject,3.2,,
function,PyObject_Calloc,3.7,,
function,PyObject_CheckBuffer,3.11,,
-function,PyObject_CheckReadBuffer,3.2,,
function,PyObject_ClearWeakRefs,3.2,,
function,PyObject_CopyData,3.11,,
+function,PyObject_DelAttr,3.13,,
+function,PyObject_DelAttrString,3.13,,
function,PyObject_DelItem,3.2,,
function,PyObject_DelItemString,3.2,,
function,PyObject_Dir,3.2,,
@@ -514,8 +524,13 @@ function,PyObject_GetAttrString,3.2,,
function,PyObject_GetBuffer,3.11,,
function,PyObject_GetItem,3.2,,
function,PyObject_GetIter,3.2,,
+function,PyObject_GetOptionalAttr,3.13,,
+function,PyObject_GetOptionalAttrString,3.13,,
+function,PyObject_GetTypeData,3.12,,
function,PyObject_HasAttr,3.2,,
function,PyObject_HasAttrString,3.2,,
+function,PyObject_HasAttrStringWithError,3.13,,
+function,PyObject_HasAttrWithError,3.13,,
function,PyObject_Hash,3.2,,
function,PyObject_HashNotImplemented,3.2,,
function,PyObject_Init,3.2,,
@@ -591,19 +606,14 @@ function,PyStructSequence_NewType,3.2,,
function,PyStructSequence_SetItem,3.2,,
var,PyStructSequence_UnnamedField,3.11,,
var,PySuper_Type,3.2,,
-function,PySys_AddWarnOption,3.2,,
-function,PySys_AddWarnOptionUnicode,3.2,,
-function,PySys_AddXOption,3.7,,
+function,PySys_Audit,3.13,,
+function,PySys_AuditTuple,3.13,,
function,PySys_FormatStderr,3.2,,
function,PySys_FormatStdout,3.2,,
function,PySys_GetObject,3.2,,
function,PySys_GetXOptions,3.7,,
-function,PySys_HasWarnOptions,3.2,,
function,PySys_ResetWarnOptions,3.2,,
-function,PySys_SetArgv,3.2,,
-function,PySys_SetArgvEx,3.2,,
function,PySys_SetObject,3.2,,
-function,PySys_SetPath,3.2,,
function,PySys_WriteStderr,3.2,,
function,PySys_WriteStdout,3.2,,
type,PyThreadState,3.2,,opaque
@@ -668,6 +678,7 @@ function,PyType_GetModuleState,3.10,,
function,PyType_GetName,3.11,,
function,PyType_GetQualName,3.11,,
function,PyType_GetSlot,3.4,,
+function,PyType_GetTypeDataSize,3.12,,
function,PyType_IsSubtype,3.2,,
function,PyType_Modified,3.2,,
function,PyType_Ready,3.2,,
@@ -750,6 +761,8 @@ function,PyUnicode_DecodeUnicodeEscape,3.2,,
function,PyUnicode_EncodeCodePage,3.7,on Windows,
function,PyUnicode_EncodeFSDefault,3.2,,
function,PyUnicode_EncodeLocale,3.7,,
+function,PyUnicode_EqualToUTF8,3.13,,
+function,PyUnicode_EqualToUTF8AndSize,3.13,,
function,PyUnicode_FSConverter,3.2,,
function,PyUnicode_FSDecoder,3.2,,
function,PyUnicode_Find,3.2,,
@@ -790,6 +803,7 @@ function,PyVectorcall_Call,3.12,,
function,PyVectorcall_NARGS,3.12,,
type,PyWeakReference,3.2,,opaque
function,PyWeakref_GetObject,3.2,,
+function,PyWeakref_GetRef,3.13,,
function,PyWeakref_NewProxy,3.2,,
function,PyWeakref_NewRef,3.2,,
var,PyWrapperDescr_Type,3.2,,
@@ -834,6 +848,7 @@ function,Py_Initialize,3.2,,
function,Py_InitializeEx,3.2,,
function,Py_Is,3.10,,
function,Py_IsFalse,3.10,,
+function,Py_IsFinalizing,3.13,,
function,Py_IsInitialized,3.2,,
function,Py_IsNone,3.10,,
function,Py_IsTrue,3.10,,
@@ -844,9 +859,6 @@ function,Py_NewInterpreter,3.2,,
function,Py_NewRef,3.10,,
function,Py_ReprEnter,3.2,,
function,Py_ReprLeave,3.2,,
-function,Py_SetPath,3.7,,
-function,Py_SetProgramName,3.2,,
-function,Py_SetPythonHome,3.2,,
function,Py_SetRecursionLimit,3.2,,
type,Py_UCS4,3.2,,
macro,Py_UNBLOCK_THREADS,3.2,,
diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst
index 2ae2726d4e4b92a..2430564b45d6d2a 100644
--- a/Doc/distributing/index.rst
+++ b/Doc/distributing/index.rst
@@ -1,179 +1,19 @@
+:orphan:
+
+.. This page is retained solely for existing links to /distributing/index.html.
+ Direct readers to the PPUG instead.
+
.. _distributing-index:
###############################
Distributing Python Modules
###############################
-:Email: distutils-sig@python.org
-
-
-As a popular open source development project, Python has an active
-supporting community of contributors and users that also make their software
-available for other Python developers to use under open source license terms.
-
-This allows Python users to share and collaborate effectively, benefiting
-from the solutions others have already created to common (and sometimes
-even rare!) problems, as well as potentially contributing their own
-solutions to the common pool.
-
-This guide covers the distribution part of the process. For a guide to
-installing other Python projects, refer to the
-:ref:`installation guide `.
-
.. note::
- For corporate and other institutional users, be aware that many
- organisations have their own policies around using and contributing to
- open source software. Please take such policies into account when making
- use of the distribution and installation tools provided with Python.
-
-
-Key terms
-=========
-
-* the `Python Package Index `__ is a public
- repository of open source licensed packages made available for use by
- other Python users
-* the `Python Packaging Authority
- `__ are the group of
- developers and documentation authors responsible for the maintenance and
- evolution of the standard packaging tools and the associated metadata and
- file format standards. They maintain a variety of tools, documentation
- and issue trackers on both `GitHub `__ and
- `Bitbucket `__.
-* ``distutils`` is the original build and distribution system first added
- to the Python standard library in 1998. While direct use of ``distutils``
- is being phased out, it still laid the foundation for the current packaging
- and distribution infrastructure, and it not only remains part of the
- standard library, but its name lives on in other ways (such as the name
- of the mailing list used to coordinate Python packaging standards
- development).
-* `setuptools`_ is a (largely) drop-in replacement for ``distutils`` first
- published in 2004. Its most notable addition over the unmodified
- ``distutils`` tools was the ability to declare dependencies on other
- packages. It is currently recommended as a more regularly updated
- alternative to ``distutils`` that offers consistent support for more
- recent packaging standards across a wide range of Python versions.
-* `wheel`_ (in this context) is a project that adds the ``bdist_wheel``
- command to ``distutils``/`setuptools`_. This produces a cross platform
- binary packaging format (called "wheels" or "wheel files" and defined in
- :pep:`427`) that allows Python libraries, even those including binary
- extensions, to be installed on a system without needing to be built
- locally.
-
-.. _setuptools: https://setuptools.readthedocs.io/en/latest/
-.. _wheel: https://wheel.readthedocs.io/
-
-Open source licensing and collaboration
-=======================================
-
-In most parts of the world, software is automatically covered by copyright.
-This means that other developers require explicit permission to copy, use,
-modify and redistribute the software.
-
-Open source licensing is a way of explicitly granting such permission in a
-relatively consistent way, allowing developers to share and collaborate
-efficiently by making common solutions to various problems freely available.
-This leaves many developers free to spend more time focusing on the problems
-that are relatively unique to their specific situation.
-
-The distribution tools provided with Python are designed to make it
-reasonably straightforward for developers to make their own contributions
-back to that common pool of software if they choose to do so.
-
-The same distribution tools can also be used to distribute software within
-an organisation, regardless of whether that software is published as open
-source software or not.
-
-
-Installing the tools
-====================
-
-The standard library does not include build tools that support modern
-Python packaging standards, as the core development team has found that it
-is important to have standard tools that work consistently, even on older
-versions of Python.
-
-The currently recommended build and distribution tools can be installed
-by invoking the ``pip`` module at the command line::
-
- python -m pip install setuptools wheel twine
-
-.. note::
-
- For POSIX users (including macOS and Linux users), these instructions
- assume the use of a :term:`virtual environment`.
-
- For Windows users, these instructions assume that the option to
- adjust the system PATH environment variable was selected when installing
- Python.
-
-The Python Packaging User Guide includes more details on the `currently
-recommended tools`_.
-
-.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations
-
-.. index::
- single: Python Package Index (PyPI)
- single: PyPI; (see Python Package Index (PyPI))
-
-.. _publishing-python-packages:
-
-Reading the Python Packaging User Guide
-=======================================
-
-The Python Packaging User Guide covers the various key steps and elements
-involved in creating and publishing a project:
-
-* `Project structure`_
-* `Building and packaging the project`_
-* `Uploading the project to the Python Package Index`_
-* `The .pypirc file`_
-
-.. _Project structure: \
- https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects
-.. _Building and packaging the project: \
- https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files
-.. _Uploading the project to the Python Package Index: \
- https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives
-.. _The .pypirc file: \
- https://packaging.python.org/specifications/pypirc/
-
-
-How do I...?
-============
-
-These are quick answers or links for some common tasks.
-
-... choose a name for my project?
----------------------------------
-
-This isn't an easy topic, but here are a few tips:
-
-* check the Python Package Index to see if the name is already in use
-* check popular hosting sites like GitHub, Bitbucket, etc to see if there
- is already a project with that name
-* check what comes up in a web search for the name you're considering
-* avoid particularly common words, especially ones with multiple meanings,
- as they can make it difficult for users to find your software when
- searching for it
-
-
-... create and distribute binary extensions?
---------------------------------------------
-
-This is actually quite a complex topic, with a variety of alternatives
-available depending on exactly what you're aiming to achieve. See the
-Python Packaging User Guide for more information and recommendations.
-
-.. seealso::
-
- `Python Packaging User Guide: Binary Extensions
- `__
-
-.. other topics:
+ Information and guidance on distributing Python modules and packages
+ has been moved to the `Python Packaging User Guide`_,
+ and the tutorial on `packaging Python projects`_.
- Once the Development & Deployment part of PPUG is fleshed out, some of
- those sections should be linked from new questions here (most notably,
- we should have a question about avoiding depending on PyPI that links to
- https://packaging.python.org/en/latest/mirrors/)
+ .. _Python Packaging User Guide: https://packaging.python.org/
+ .. _packaging Python projects: https://packaging.python.org/en/latest/tutorials/packaging-projects/
diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst
index 880bb33ee56718b..ddde567f6f3efa5 100644
--- a/Doc/extending/building.rst
+++ b/Doc/extending/building.rst
@@ -45,6 +45,7 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details.
.. highlight:: c
+.. _install-index:
.. _setuptools-index:
Building C and C++ Extensions with setuptools
diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst
index e64db3733440383..20397dc5add5db6 100644
--- a/Doc/extending/embedding.rst
+++ b/Doc/extending/embedding.rst
@@ -59,24 +59,43 @@ perform some operation on a file. ::
int
main(int argc, char *argv[])
{
- wchar_t *program = Py_DecodeLocale(argv[0], NULL);
- if (program == NULL) {
- fprintf(stderr, "Fatal error: cannot decode argv[0]\n");
- exit(1);
+ PyStatus status;
+ PyConfig config;
+ PyConfig_InitPythonConfig(&config);
+
+ /* optional but recommended */
+ status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]);
+ if (PyStatus_Exception(status)) {
+ goto exception;
}
- Py_SetProgramName(program); /* optional but recommended */
- Py_Initialize();
+
+ status = Py_InitializeFromConfig(&config);
+ if (PyStatus_Exception(status)) {
+ goto exception;
+ }
+ PyConfig_Clear(&config);
+
PyRun_SimpleString("from time import time,ctime\n"
"print('Today is', ctime(time()))\n");
if (Py_FinalizeEx() < 0) {
exit(120);
}
- PyMem_RawFree(program);
return 0;
+
+ exception:
+ PyConfig_Clear(&config);
+ Py_ExitStatusException(status);
}
-The :c:func:`Py_SetProgramName` function should be called before
-:c:func:`Py_Initialize` to inform the interpreter about paths to Python run-time
+.. note::
+
+ ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be
+ used in some APIs instead of ``int``.
+ It is not necessary since Python 3.13, but we keep it here for backward compatibility.
+ See :ref:`arg-parsing-string-and-buffers` for a description of this macro.
+
+Setting :c:member:`PyConfig.program_name` should be called before
+:c:func:`Py_InitializeFromConfig` to inform the interpreter about paths to Python run-time
libraries. Next, the Python interpreter is initialized with
:c:func:`Py_Initialize`, followed by the execution of a hard-coded Python script
that prints the date and time. Afterwards, the :c:func:`Py_FinalizeEx` call shuts
@@ -250,7 +269,7 @@ following two statements before the call to :c:func:`Py_Initialize`::
PyImport_AppendInittab("emb", &PyInit_emb);
These two lines initialize the ``numargs`` variable, and make the
-:func:`emb.numargs` function accessible to the embedded Python interpreter.
+:func:`!emb.numargs` function accessible to the embedded Python interpreter.
With these extensions, the Python script can do things like
.. code-block:: python
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index d9bf4fd6c7ae0e3..1ee7f28b2ba2206 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -69,8 +69,10 @@ the module and a copyright notice if you like).
headers on some systems, you *must* include :file:`Python.h` before any standard
headers are included.
- It is recommended to always define ``PY_SSIZE_T_CLEAN`` before including
- ``Python.h``. See :ref:`parsetuple` for a description of this macro.
+ ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be
+ used in some APIs instead of ``int``.
+ It is not necessary since Python 3.13, but we keep it here for backward compatibility.
+ See :ref:`arg-parsing-string-and-buffers` for a description of this macro.
All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or
``PY``, except those defined in standard header files. For convenience, and
@@ -195,7 +197,7 @@ The choice of which exception to raise is entirely yours. There are predeclared
C objects corresponding to all built-in Python exceptions, such as
:c:data:`PyExc_ZeroDivisionError`, which you can use directly. Of course, you
should choose exceptions wisely --- don't use :c:data:`PyExc_TypeError` to mean
-that a file couldn't be opened (that should probably be :c:data:`PyExc_IOError`).
+that a file couldn't be opened (that should probably be :c:data:`PyExc_OSError`).
If something's wrong with the argument list, the :c:func:`PyArg_ParseTuple`
function usually raises :c:data:`PyExc_TypeError`. If you have an argument whose
value must be in a particular range or must satisfy other conditions,
@@ -206,7 +208,7 @@ usually declare a static object variable at the beginning of your file::
static PyObject *SpamError;
-and initialize it in your module's initialization function (:c:func:`PyInit_spam`)
+and initialize it in your module's initialization function (:c:func:`!PyInit_spam`)
with an exception object::
PyMODINIT_FUNC
@@ -219,9 +221,7 @@ with an exception object::
return NULL;
SpamError = PyErr_NewException("spam.error", NULL, NULL);
- Py_XINCREF(SpamError);
- if (PyModule_AddObject(m, "error", SpamError) < 0) {
- Py_XDECREF(SpamError);
+ if (PyModule_AddObjectRef(m, "error", SpamError) < 0) {
Py_CLEAR(SpamError);
Py_DECREF(m);
return NULL;
@@ -230,22 +230,22 @@ with an exception object::
return m;
}
-Note that the Python name for the exception object is :exc:`spam.error`. The
+Note that the Python name for the exception object is :exc:`!spam.error`. The
:c:func:`PyErr_NewException` function may create a class with the base class
being :exc:`Exception` (unless another class is passed in instead of ``NULL``),
described in :ref:`bltin-exceptions`.
-Note also that the :c:data:`SpamError` variable retains a reference to the newly
+Note also that the :c:data:`!SpamError` variable retains a reference to the newly
created exception class; this is intentional! Since the exception could be
removed from the module by external code, an owned reference to the class is
-needed to ensure that it will not be discarded, causing :c:data:`SpamError` to
+needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to
become a dangling pointer. Should it become a dangling pointer, C code which
raises the exception could cause a core dump or other unintended side effects.
-We discuss the use of ``PyMODINIT_FUNC`` as a function return type later in this
+We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this
sample.
-The :exc:`spam.error` exception can be raised in your extension module using a
+The :exc:`!spam.error` exception can be raised in your extension module using a
call to :c:func:`PyErr_SetString` as shown below::
static PyObject *
@@ -279,9 +279,9 @@ statement::
It returns ``NULL`` (the error indicator for functions returning object pointers)
if an error is detected in the argument list, relying on the exception set by
:c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been
-copied to the local variable :c:data:`command`. This is a pointer assignment and
+copied to the local variable :c:data:`!command`. This is a pointer assignment and
you are not supposed to modify the string to which it points (so in Standard C,
-the variable :c:data:`command` should properly be declared as ``const char
+the variable :c:data:`!command` should properly be declared as ``const char
*command``).
The next statement is a call to the Unix function :c:func:`system`, passing it
@@ -289,7 +289,7 @@ the string we just got from :c:func:`PyArg_ParseTuple`::
sts = system(command);
-Our :func:`spam.system` function must return the value of :c:data:`sts` as a
+Our :func:`!spam.system` function must return the value of :c:data:`!sts` as a
Python object. This is done using the function :c:func:`PyLong_FromLong`. ::
return PyLong_FromLong(sts);
@@ -315,7 +315,7 @@ contexts, as we have seen.
The Module's Method Table and Initialization Function
=====================================================
-I promised to show how :c:func:`spam_system` is called from Python programs.
+I promised to show how :c:func:`!spam_system` is called from Python programs.
First, we need to list its name and address in a "method table"::
static PyMethodDef SpamMethods[] = {
@@ -335,7 +335,7 @@ When using only ``METH_VARARGS``, the function should expect the Python-level
parameters to be passed in as a tuple acceptable for parsing via
:c:func:`PyArg_ParseTuple`; more information on this function is provided below.
-The :const:`METH_KEYWORDS` bit may be set in the third field if keyword
+The :c:macro:`METH_KEYWORDS` bit may be set in the third field if keyword
arguments should be passed to the function. In this case, the C function should
accept a third ``PyObject *`` parameter which will be a dictionary of keywords.
Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a
@@ -354,7 +354,7 @@ The method table must be referenced in the module definition structure::
This structure, in turn, must be passed to the interpreter in the module's
initialization function. The initialization function must be named
-:c:func:`PyInit_name`, where *name* is the name of the module, and should be the
+:c:func:`!PyInit_name`, where *name* is the name of the module, and should be the
only non-\ ``static`` item defined in the module file::
PyMODINIT_FUNC
@@ -363,12 +363,12 @@ only non-\ ``static`` item defined in the module file::
return PyModule_Create(&spammodule);
}
-Note that PyMODINIT_FUNC declares the function as ``PyObject *`` return type,
+Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type,
declares any special linkage declarations required by the platform, and for C++
declares the function as ``extern "C"``.
-When the Python program imports module :mod:`spam` for the first time,
-:c:func:`PyInit_spam` is called. (See below for comments about embedding Python.)
+When the Python program imports module :mod:`!spam` for the first time,
+:c:func:`!PyInit_spam` is called. (See below for comments about embedding Python.)
It calls :c:func:`PyModule_Create`, which returns a module object, and
inserts built-in function objects into the newly created module based upon the
table (an array of :c:type:`PyMethodDef` structures) found in the module definition.
@@ -378,19 +378,20 @@ certain errors, or return ``NULL`` if the module could not be initialized
satisfactorily. The init function must return the module object to its caller,
so that it then gets inserted into ``sys.modules``.
-When embedding Python, the :c:func:`PyInit_spam` function is not called
+When embedding Python, the :c:func:`!PyInit_spam` function is not called
automatically unless there's an entry in the :c:data:`PyImport_Inittab` table.
To add the module to the initialization table, use :c:func:`PyImport_AppendInittab`,
optionally followed by an import of the module::
+ #define PY_SSIZE_T_CLEAN
+ #include
+
int
main(int argc, char *argv[])
{
- wchar_t *program = Py_DecodeLocale(argv[0], NULL);
- if (program == NULL) {
- fprintf(stderr, "Fatal error: cannot decode argv[0]\n");
- exit(1);
- }
+ PyStatus status;
+ PyConfig config;
+ PyConfig_InitPythonConfig(&config);
/* Add a built-in module, before Py_Initialize */
if (PyImport_AppendInittab("spam", PyInit_spam) == -1) {
@@ -399,11 +400,18 @@ optionally followed by an import of the module::
}
/* Pass argv[0] to the Python interpreter */
- Py_SetProgramName(program);
+ status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]);
+ if (PyStatus_Exception(status)) {
+ goto exception;
+ }
/* Initialize the Python interpreter. Required.
If this step fails, it will be a fatal error. */
- Py_Initialize();
+ status = Py_InitializeFromConfig(&config);
+ if (PyStatus_Exception(status)) {
+ goto exception;
+ }
+ PyConfig_Clear(&config);
/* Optionally import the module; alternatively,
import can be deferred until the embedded script
@@ -414,10 +422,13 @@ optionally followed by an import of the module::
fprintf(stderr, "Error: could not import module 'spam'\n");
}
- ...
+ // ... use Python C API here ...
- PyMem_RawFree(program);
return 0;
+
+ exception:
+ PyConfig_Clear(&config);
+ Py_ExitStatusException(status);
}
.. note::
@@ -527,7 +538,7 @@ be part of a module definition::
}
This function must be registered with the interpreter using the
-:const:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The
+:c:macro:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The
:c:func:`PyArg_ParseTuple` function and its arguments are documented in section
:ref:`parsetuple`.
@@ -649,7 +660,7 @@ Note that any Python object references which are provided to the caller are
Some example calls::
- #define PY_SSIZE_T_CLEAN /* Make "s#" use Py_ssize_t rather than int. */
+ #define PY_SSIZE_T_CLEAN
#include
::
@@ -745,7 +756,7 @@ it returns false and raises an appropriate exception.
Here is an example module which uses keywords, based on an example by Geoff
Philbrick (philbrick@hks.com)::
- #define PY_SSIZE_T_CLEAN /* Make "s#" use Py_ssize_t rather than int. */
+ #define PY_SSIZE_T_CLEAN
#include
static PyObject *
@@ -1030,13 +1041,13 @@ Let's follow the control flow into :c:func:`PyList_SetItem`. The list owns
references to all its items, so when item 1 is replaced, it has to dispose of
the original item 1. Now let's suppose the original item 1 was an instance of a
user-defined class, and let's further suppose that the class defined a
-:meth:`__del__` method. If this class instance has a reference count of 1,
-disposing of it will call its :meth:`__del__` method.
+:meth:`!__del__` method. If this class instance has a reference count of 1,
+disposing of it will call its :meth:`!__del__` method.
-Since it is written in Python, the :meth:`__del__` method can execute arbitrary
+Since it is written in Python, the :meth:`!__del__` method can execute arbitrary
Python code. Could it perhaps do something to invalidate the reference to
-``item`` in :c:func:`bug`? You bet! Assuming that the list passed into
-:c:func:`bug` is accessible to the :meth:`__del__` method, it could execute a
+``item`` in :c:func:`!bug`? You bet! Assuming that the list passed into
+:c:func:`!bug` is accessible to the :meth:`!__del__` method, it could execute a
statement to the effect of ``del list[0]``, and assuming this was the last
reference to that object, it would free the memory associated with it, thereby
invalidating ``item``.
@@ -1057,7 +1068,7 @@ increment the reference count. The correct version of the function reads::
This is a true story. An older version of Python contained variants of this bug
and someone spent a considerable amount of time in a C debugger to figure out
-why his :meth:`__del__` methods would fail...
+why his :meth:`!__del__` methods would fail...
The second case of problems with a borrowed reference is a variant involving
threads. Normally, multiple threads in the Python interpreter can't get in each
@@ -1208,14 +1219,14 @@ file corresponding to the module provides a macro that takes care of importing
the module and retrieving its C API pointers; client modules only have to call
this macro before accessing the C API.
-The exporting module is a modification of the :mod:`spam` module from section
-:ref:`extending-simpleexample`. The function :func:`spam.system` does not call
+The exporting module is a modification of the :mod:`!spam` module from section
+:ref:`extending-simpleexample`. The function :func:`!spam.system` does not call
the C library function :c:func:`system` directly, but a function
-:c:func:`PySpam_System`, which would of course do something more complicated in
+:c:func:`!PySpam_System`, which would of course do something more complicated in
reality (such as adding "spam" to every command). This function
-:c:func:`PySpam_System` is also exported to other extension modules.
+:c:func:`!PySpam_System` is also exported to other extension modules.
-The function :c:func:`PySpam_System` is a plain C function, declared
+The function :c:func:`!PySpam_System` is a plain C function, declared
``static`` like everything else::
static int
@@ -1224,7 +1235,7 @@ The function :c:func:`PySpam_System` is a plain C function, declared
return system(command);
}
-The function :c:func:`spam_system` is modified in a trivial way::
+The function :c:func:`!spam_system` is modified in a trivial way::
static PyObject *
spam_system(PyObject *self, PyObject *args)
@@ -1268,8 +1279,7 @@ function must take care of initializing the C API pointer array::
/* Create a Capsule containing the API pointer array's address */
c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL);
- if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) {
- Py_XDECREF(c_api_object);
+ if (PyModule_Add(m, "_C_API", c_api_object) < 0) {
Py_DECREF(m);
return NULL;
}
@@ -1278,7 +1288,7 @@ function must take care of initializing the C API pointer array::
}
Note that ``PySpam_API`` is declared ``static``; otherwise the pointer
-array would disappear when :func:`PyInit_spam` terminates!
+array would disappear when :c:func:`!PyInit_spam` terminates!
The bulk of the work is in the header file :file:`spammodule.h`, which looks
like this::
@@ -1332,8 +1342,8 @@ like this::
#endif /* !defined(Py_SPAMMODULE_H) */
All that a client module must do in order to have access to the function
-:c:func:`PySpam_System` is to call the function (or rather macro)
-:c:func:`import_spam` in its initialization function::
+:c:func:`!PySpam_System` is to call the function (or rather macro)
+:c:func:`!import_spam` in its initialization function::
PyMODINIT_FUNC
PyInit_client(void)
diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index 3de849ade788887..7a92b3257c6cd31 100644
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -149,7 +149,7 @@ done. This can be done using the :c:func:`PyErr_Fetch` and
.. index::
single: string; object representation
- builtin: repr
+ pair: built-in function; repr
Object Presentation
-------------------
@@ -168,7 +168,7 @@ representation of the instance for which it is called. Here is a simple
example::
static PyObject *
- newdatatype_repr(newdatatypeobject * obj)
+ newdatatype_repr(newdatatypeobject *obj)
{
return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}",
obj->obj_UnderlyingDatatypePtr->size);
@@ -188,7 +188,7 @@ used instead.
Here is a simple example::
static PyObject *
- newdatatype_str(newdatatypeobject * obj)
+ newdatatype_str(newdatatypeobject *obj)
{
return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}",
obj->obj_UnderlyingDatatypePtr->size);
@@ -270,7 +270,7 @@ structure::
One entry should be defined for each method provided by the type; no entries are
needed for methods inherited from a base type. One additional entry is needed
at the end; it is a sentinel that marks the end of the array. The
-:attr:`ml_name` field of the sentinel must be ``NULL``.
+:c:member:`~PyMethodDef.ml_name` field of the sentinel must be ``NULL``.
The second table is used to define attributes which map directly to data stored
in the instance. A variety of primitive C types are supported, and access may
@@ -286,44 +286,19 @@ be read-only or read-write. The structures in the table are defined as::
For each entry in the table, a :term:`descriptor` will be constructed and added to the
type which will be able to extract a value from the instance structure. The
-:attr:`type` field should contain one of the type codes defined in the
-:file:`structmember.h` header; the value will be used to determine how to
-convert Python values to and from C values. The :attr:`flags` field is used to
-store flags which control how the attribute can be accessed.
-
-The following flag constants are defined in :file:`structmember.h`; they may be
-combined using bitwise-OR.
-
-+---------------------------+----------------------------------------------+
-| Constant | Meaning |
-+===========================+==============================================+
-| :const:`READONLY` | Never writable. |
-+---------------------------+----------------------------------------------+
-| :const:`PY_AUDIT_READ` | Emit an ``object.__getattr__`` |
-| | :ref:`audit events ` before |
-| | reading. |
-+---------------------------+----------------------------------------------+
-
-.. versionchanged:: 3.10
- :const:`RESTRICTED`, :const:`READ_RESTRICTED` and :const:`WRITE_RESTRICTED`
- are deprecated. However, :const:`READ_RESTRICTED` is an alias for
- :const:`PY_AUDIT_READ`, so fields that specify either :const:`RESTRICTED`
- or :const:`READ_RESTRICTED` will also raise an audit event.
-
-.. index::
- single: READONLY
- single: READ_RESTRICTED
- single: WRITE_RESTRICTED
- single: RESTRICTED
- single: PY_AUDIT_READ
+:c:member:`~PyMemberDef.type` field should contain a type code like :c:macro:`Py_T_INT` or
+:c:macro:`Py_T_DOUBLE`; the value will be used to determine how to
+convert Python values to and from C values. The :c:member:`~PyMemberDef.flags` field is used to
+store flags which control how the attribute can be accessed: you can set it to
+:c:macro:`Py_READONLY` to prevent Python code from setting it.
An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table to build
descriptors that are used at runtime is that any attribute defined this way can
have an associated doc string simply by providing the text in the table. An
application can use the introspection API to retrieve the descriptor from the
-class object, and get the doc string using its :attr:`__doc__` attribute.
+class object, and get the doc string using its :attr:`!__doc__` attribute.
-As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :attr:`name` value
+As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value
of ``NULL`` is required.
.. XXX Descriptors need to be explained in more detail somewhere, but not here.
@@ -348,7 +323,7 @@ called, so that if you do need to extend their functionality, you'll understand
what needs to be done.
The :c:member:`~PyTypeObject.tp_getattr` handler is called when the object requires an attribute
-look-up. It is called in the same situations where the :meth:`__getattr__`
+look-up. It is called in the same situations where the :meth:`~object.__getattr__`
method of a class would be called.
Here is an example::
@@ -362,13 +337,13 @@ Here is an example::
}
PyErr_Format(PyExc_AttributeError,
- "'%.50s' object has no attribute '%.400s'",
- tp->tp_name, name);
+ "'%.100s' object has no attribute '%.400s'",
+ Py_TYPE(obj)->tp_name, name);
return NULL;
}
-The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`__setattr__` or
-:meth:`__delattr__` method of a class instance would be called. When an
+The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`~object.__setattr__` or
+:meth:`~object.__delattr__` method of a class instance would be called. When an
attribute should be deleted, the third parameter will be ``NULL``. Here is an
example that simply raises an exception; if this were really all you wanted, the
:c:member:`~PyTypeObject.tp_setattr` handler should be set to ``NULL``. ::
@@ -389,7 +364,7 @@ Object Comparison
The :c:member:`~PyTypeObject.tp_richcompare` handler is called when comparisons are needed. It is
analogous to the :ref:`rich comparison methods `, like
-:meth:`__lt__`, and also called by :c:func:`PyObject_RichCompare` and
+:meth:`!__lt__`, and also called by :c:func:`PyObject_RichCompare` and
:c:func:`PyObject_RichCompareBool`.
This function is called with two Python objects and the operator as arguments,
@@ -404,7 +379,7 @@ Here is a sample implementation, for a datatype that is considered equal if the
size of an internal pointer is equal::
static PyObject *
- newdatatype_richcmp(PyObject *obj1, PyObject *obj2, int op)
+ newdatatype_richcmp(newdatatypeobject *obj1, newdatatypeobject *obj2, int op)
{
PyObject *result;
int c, size1, size2;
@@ -503,7 +478,7 @@ This function takes three arguments:
Here is a toy ``tp_call`` implementation::
static PyObject *
- newdatatype_call(newdatatypeobject *self, PyObject *args, PyObject *kwds)
+ newdatatype_call(newdatatypeobject *obj, PyObject *args, PyObject *kwds)
{
PyObject *result;
const char *arg1;
@@ -530,7 +505,7 @@ These functions provide support for the iterator protocol. Both handlers
take exactly one parameter, the instance for which they are being called,
and return a new reference. In the case of an error, they should set an
exception and return ``NULL``. :c:member:`~PyTypeObject.tp_iter` corresponds
-to the Python :meth:`__iter__` method, while :c:member:`~PyTypeObject.tp_iternext`
+to the Python :meth:`~object.__iter__` method, while :c:member:`~PyTypeObject.tp_iternext`
corresponds to the Python :meth:`~iterator.__next__` method.
Any :term:`iterable` object must implement the :c:member:`~PyTypeObject.tp_iter`
diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst
index 5d4a3f06dd54026..7eba9759119b3bf 100644
--- a/Doc/extending/newtypes_tutorial.rst
+++ b/Doc/extending/newtypes_tutorial.rst
@@ -36,8 +36,8 @@ So, if you want to define a new extension type, you need to create a new type
object.
This sort of thing can only be explained by example, so here's a minimal, but
-complete, module that defines a new type named :class:`Custom` inside a C
-extension module :mod:`custom`:
+complete, module that defines a new type named :class:`!Custom` inside a C
+extension module :mod:`!custom`:
.. note::
What we're showing here is the traditional way of defining *static*
@@ -45,17 +45,17 @@ extension module :mod:`custom`:
allows defining heap-allocated extension types using the
:c:func:`PyType_FromSpec` function, which isn't covered in this tutorial.
-.. literalinclude:: ../includes/custom.c
+.. literalinclude:: ../includes/newtypes/custom.c
Now that's quite a bit to take in at once, but hopefully bits will seem familiar
from the previous chapter. This file defines three things:
-#. What a :class:`Custom` **object** contains: this is the ``CustomObject``
- struct, which is allocated once for each :class:`Custom` instance.
-#. How the :class:`Custom` **type** behaves: this is the ``CustomType`` struct,
+#. What a :class:`!Custom` **object** contains: this is the ``CustomObject``
+ struct, which is allocated once for each :class:`!Custom` instance.
+#. How the :class:`!Custom` **type** behaves: this is the ``CustomType`` struct,
which defines a set of flags and function pointers that the interpreter
inspects when specific operations are requested.
-#. How to initialize the :mod:`custom` module: this is the ``PyInit_custom``
+#. How to initialize the :mod:`!custom` module: this is the ``PyInit_custom``
function and the associated ``custommodule`` struct.
The first bit is::
@@ -88,7 +88,7 @@ standard Python floats::
The second bit is the definition of the type object. ::
static PyTypeObject CustomType = {
- PyVarObject_HEAD_INIT(NULL, 0)
+ .ob_base = PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom.Custom",
.tp_doc = PyDoc_STR("Custom objects"),
.tp_basicsize = sizeof(CustomObject),
@@ -109,7 +109,7 @@ common practice to not specify them explicitly unless you need them.
We're going to pick it apart, one field at a time::
- PyVarObject_HEAD_INIT(NULL, 0)
+ .ob_base = PyVarObject_HEAD_INIT(NULL, 0)
This line is mandatory boilerplate to initialize the ``ob_base``
field mentioned above. ::
@@ -127,8 +127,8 @@ our objects and in some error messages, for example:
TypeError: can only concatenate str (not "custom.Custom") to str
Note that the name is a dotted name that includes both the module name and the
-name of the type within the module. The module in this case is :mod:`custom` and
-the type is :class:`Custom`, so we set the type name to :class:`custom.Custom`.
+name of the type within the module. The module in this case is :mod:`!custom` and
+the type is :class:`!Custom`, so we set the type name to :class:`!custom.Custom`.
Using the real dotted import path is important to make your type compatible
with the :mod:`pydoc` and :mod:`pickle` modules. ::
@@ -136,7 +136,7 @@ with the :mod:`pydoc` and :mod:`pickle` modules. ::
.tp_itemsize = 0,
This is so that Python knows how much memory to allocate when creating
-new :class:`Custom` instances. :c:member:`~PyTypeObject.tp_itemsize` is
+new :class:`!Custom` instances. :c:member:`~PyTypeObject.tp_itemsize` is
only used for variable-sized objects and should otherwise be zero.
.. note::
@@ -145,13 +145,13 @@ only used for variable-sized objects and should otherwise be zero.
:c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple
inheritance. A Python subclass of your type will have to list your type first
in its :attr:`~class.__bases__`, or else it will not be able to call your type's
- :meth:`__new__` method without getting an error. You can avoid this problem by
+ :meth:`~object.__new__` method without getting an error. You can avoid this problem by
ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its
base type does. Most of the time, this will be true anyway, because either your
base type will be :class:`object`, or else you will be adding data members to
your base type, and therefore increasing its size.
-We set the class flags to :const:`Py_TPFLAGS_DEFAULT`. ::
+We set the class flags to :c:macro:`Py_TPFLAGS_DEFAULT`. ::
.tp_flags = Py_TPFLAGS_DEFAULT,
@@ -164,31 +164,29 @@ We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. ::
.tp_doc = PyDoc_STR("Custom objects"),
To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new`
-handler. This is the equivalent of the Python method :meth:`__new__`, but
+handler. This is the equivalent of the Python method :meth:`~object.__new__`, but
has to be specified explicitly. In this case, we can just use the default
implementation provided by the API function :c:func:`PyType_GenericNew`. ::
.tp_new = PyType_GenericNew,
Everything else in the file should be familiar, except for some code in
-:c:func:`PyInit_custom`::
+:c:func:`!PyInit_custom`::
if (PyType_Ready(&CustomType) < 0)
return;
-This initializes the :class:`Custom` type, filling in a number of members
-to the appropriate default values, including :attr:`ob_type` that we initially
+This initializes the :class:`!Custom` type, filling in a number of members
+to the appropriate default values, including :c:member:`~PyObject.ob_type` that we initially
set to ``NULL``. ::
- Py_INCREF(&CustomType);
- if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
- Py_DECREF(&CustomType);
+ if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
Py_DECREF(m);
return NULL;
}
This adds the type to the module dictionary. This allows us to create
-:class:`Custom` instances by calling the :class:`Custom` class:
+:class:`!Custom` instances by calling the :class:`!Custom` class:
.. code-block:: pycon
@@ -196,57 +194,46 @@ This adds the type to the module dictionary. This allows us to create
>>> mycustom = custom.Custom()
That's it! All that remains is to build it; put the above code in a file called
-:file:`custom.c` and:
+:file:`custom.c`,
+
+.. literalinclude:: ../includes/newtypes/pyproject.toml
+
+in a file called :file:`pyproject.toml`, and
.. code-block:: python
- from distutils.core import setup, Extension
- setup(name="custom", version="1.0",
- ext_modules=[Extension("custom", ["custom.c"])])
+ from setuptools import Extension, setup
+ setup(ext_modules=[Extension("custom", ["custom.c"])])
in a file called :file:`setup.py`; then typing
.. code-block:: shell-session
- $ python setup.py build
+ $ python -m pip install .
-at a shell should produce a file :file:`custom.so` in a subdirectory; move to
-that directory and fire up Python --- you should be able to ``import custom`` and
-play around with Custom objects.
+in a shell should produce a file :file:`custom.so` in a subdirectory
+and install it; now fire up Python --- you should be able to ``import custom``
+and play around with ``Custom`` objects.
That wasn't so hard, was it?
Of course, the current Custom type is pretty uninteresting. It has no data and
doesn't do anything. It can't even be subclassed.
-.. note::
- While this documentation showcases the standard :mod:`distutils` module
- for building C extensions, it is recommended in real-world use cases to
- use the newer and better-maintained ``setuptools`` library. Documentation
- on how to do this is out of scope for this document and can be found in
- the `Python Packaging User's Guide `_.
-
Adding data and methods to the Basic example
============================================
Let's extend the basic example to add some data and methods. Let's also make
-the type usable as a base class. We'll create a new module, :mod:`custom2` that
+the type usable as a base class. We'll create a new module, :mod:`!custom2` that
adds these capabilities:
-.. literalinclude:: ../includes/custom2.c
+.. literalinclude:: ../includes/newtypes/custom2.c
This version of the module has a number of changes.
-We've added an extra include::
-
- #include
-
-This include provides declarations that we use to handle attributes, as
-described a bit later.
-
-The :class:`Custom` type now has three data attributes in its C struct,
+The :class:`!Custom` type now has three data attributes in its C struct,
*first*, *last*, and *number*. The *first* and *last* variables are Python
strings containing first and last names. The *number* attribute is a C integer.
@@ -279,7 +266,7 @@ This method first clears the reference counts of the two Python attributes.
``NULL`` (which might happen here if ``tp_new`` failed midway). It then
calls the :c:member:`~PyTypeObject.tp_free` member of the object's type
(computed by ``Py_TYPE(self)``) to free the object's memory. Note that
-the object's type might not be :class:`CustomType`, because the object may
+the object's type might not be :class:`!CustomType`, because the object may
be an instance of a subclass.
.. note::
@@ -318,10 +305,10 @@ and install it in the :c:member:`~PyTypeObject.tp_new` member::
.tp_new = Custom_new,
The ``tp_new`` handler is responsible for creating (as opposed to initializing)
-objects of the type. It is exposed in Python as the :meth:`__new__` method.
+objects of the type. It is exposed in Python as the :meth:`~object.__new__` method.
It is not required to define a ``tp_new`` member, and indeed many extension
types will simply reuse :c:func:`PyType_GenericNew` as done in the first
-version of the ``Custom`` type above. In this case, we use the ``tp_new``
+version of the :class:`!Custom` type above. In this case, we use the ``tp_new``
handler to initialize the ``first`` and ``last`` attributes to non-``NULL``
default values.
@@ -352,7 +339,7 @@ result against ``NULL`` before proceeding.
.. note::
If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one
- that calls a base type's :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`),
+ that calls a base type's :c:member:`~PyTypeObject.tp_new` or :meth:`~object.__new__`),
you must *not* try to determine what method to call using method resolution
order at runtime. Always statically determine what type you are going to
call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via
@@ -395,14 +382,14 @@ by filling the :c:member:`~PyTypeObject.tp_init` slot. ::
.tp_init = (initproc) Custom_init,
The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the
-:meth:`__init__` method. It is used to initialize an object after it's
+:meth:`~object.__init__` method. It is used to initialize an object after it's
created. Initializers always accept positional and keyword arguments,
and they should return either ``0`` on success or ``-1`` on error.
Unlike the ``tp_new`` handler, there is no guarantee that ``tp_init``
is called at all (for example, the :mod:`pickle` module by default
-doesn't call :meth:`__init__` on unpickled instances). It can also be
-called multiple times. Anyone can call the :meth:`__init__` method on
+doesn't call :meth:`~object.__init__` on unpickled instances). It can also be
+called multiple times. Anyone can call the :meth:`!__init__` method on
our objects. For this reason, we have to be extra careful when assigning
the new attribute values. We might be tempted, for example to assign the
``first`` member like this::
@@ -436,11 +423,11 @@ We want to expose our instance variables as attributes. There are a
number of ways to do that. The simplest way is to define member definitions::
static PyMemberDef Custom_members[] = {
- {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0,
+ {"first", Py_T_OBJECT_EX, offsetof(CustomObject, first), 0,
"first name"},
- {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0,
+ {"last", Py_T_OBJECT_EX, offsetof(CustomObject, last), 0,
"last name"},
- {"number", T_INT, offsetof(CustomObject, number), 0,
+ {"number", Py_T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
@@ -460,7 +447,7 @@ Further, the attributes can be deleted, setting the C pointers to ``NULL``. Eve
though we can make sure the members are initialized to non-``NULL`` values, the
members can be set to ``NULL`` if the attributes are deleted.
-We define a single method, :meth:`Custom.name()`, that outputs the objects name as the
+We define a single method, :meth:`!Custom.name()`, that outputs the objects name as the
concatenation of the first and last names. ::
static PyObject *
@@ -477,8 +464,8 @@ concatenation of the first and last names. ::
return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
-The method is implemented as a C function that takes a :class:`Custom` (or
-:class:`Custom` subclass) instance as the first argument. Methods always take an
+The method is implemented as a C function that takes a :class:`!Custom` (or
+:class:`!Custom` subclass) instance as the first argument. Methods always take an
instance as the first argument. Methods often take positional and keyword
arguments as well, but in this case we don't take any and don't need to accept
a positional argument tuple or keyword argument dictionary. This method is
@@ -489,8 +476,8 @@ equivalent to the Python method:
def name(self):
return "%s %s" % (self.first, self.last)
-Note that we have to check for the possibility that our :attr:`first` and
-:attr:`last` members are ``NULL``. This is because they can be deleted, in which
+Note that we have to check for the possibility that our :attr:`!first` and
+:attr:`!last` members are ``NULL``. This is because they can be deleted, in which
case they are set to ``NULL``. It would be better to prevent deletion of these
attributes and to restrict the attribute values to be strings. We'll see how to
do that in the next section.
@@ -505,7 +492,7 @@ definitions::
{NULL} /* Sentinel */
};
-(note that we used the :const:`METH_NOARGS` flag to indicate that the method
+(note that we used the :c:macro:`METH_NOARGS` flag to indicate that the method
is expecting no arguments other than *self*)
and assign it to the :c:member:`~PyTypeObject.tp_methods` slot::
@@ -515,41 +502,45 @@ and assign it to the :c:member:`~PyTypeObject.tp_methods` slot::
Finally, we'll make our type usable as a base class for subclassing. We've
written our methods carefully so far so that they don't make any assumptions
about the type of the object being created or used, so all we need to do is
-to add the :const:`Py_TPFLAGS_BASETYPE` to our class flag definition::
+to add the :c:macro:`Py_TPFLAGS_BASETYPE` to our class flag definition::
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
-We rename :c:func:`PyInit_custom` to :c:func:`PyInit_custom2`, update the
+We rename :c:func:`!PyInit_custom` to :c:func:`!PyInit_custom2`, update the
module name in the :c:type:`PyModuleDef` struct, and update the full class
name in the :c:type:`PyTypeObject` struct.
-Finally, we update our :file:`setup.py` file to build the new module:
+Finally, we update our :file:`setup.py` file to include the new module,
.. code-block:: python
- from distutils.core import setup, Extension
- setup(name="custom", version="1.0",
- ext_modules=[
- Extension("custom", ["custom.c"]),
- Extension("custom2", ["custom2.c"]),
- ])
+ from setuptools import Extension, setup
+ setup(ext_modules=[
+ Extension("custom", ["custom.c"]),
+ Extension("custom2", ["custom2.c"]),
+ ])
+
+and then we re-install so that we can ``import custom2``:
+
+.. code-block:: shell-session
+ $ python -m pip install .
Providing finer control over data attributes
============================================
-In this section, we'll provide finer control over how the :attr:`first` and
-:attr:`last` attributes are set in the :class:`Custom` example. In the previous
-version of our module, the instance variables :attr:`first` and :attr:`last`
+In this section, we'll provide finer control over how the :attr:`!first` and
+:attr:`!last` attributes are set in the :class:`!Custom` example. In the previous
+version of our module, the instance variables :attr:`!first` and :attr:`!last`
could be set to non-string values or even deleted. We want to make sure that
these attributes always contain strings.
-.. literalinclude:: ../includes/custom3.c
+.. literalinclude:: ../includes/newtypes/custom3.c
-To provide greater control, over the :attr:`first` and :attr:`last` attributes,
+To provide greater control, over the :attr:`!first` and :attr:`!last` attributes,
we'll use custom getter and setter functions. Here are the functions for
-getting and setting the :attr:`first` attribute::
+getting and setting the :attr:`!first` attribute::
static PyObject *
Custom_getfirst(CustomObject *self, void *closure)
@@ -578,13 +569,13 @@ getting and setting the :attr:`first` attribute::
return 0;
}
-The getter function is passed a :class:`Custom` object and a "closure", which is
+The getter function is passed a :class:`!Custom` object and a "closure", which is
a void pointer. In this case, the closure is ignored. (The closure supports an
advanced usage in which definition data is passed to the getter and setter. This
could, for example, be used to allow a single set of getter and setter functions
that decide the attribute to get or set based on data in the closure.)
-The setter function is passed the :class:`Custom` object, the new value, and the
+The setter function is passed the :class:`!Custom` object, the new value, and the
closure. The new value may be ``NULL``, in which case the attribute is being
deleted. In our setter, we raise an error if the attribute is deleted or if its
new value is not a string.
@@ -609,7 +600,7 @@ above. In this case, we aren't using a closure, so we just pass ``NULL``.
We also remove the member definitions for these attributes::
static PyMemberDef Custom_members[] = {
- {"number", T_INT, offsetof(CustomObject, number), 0,
+ {"number", Py_T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
@@ -673,11 +664,11 @@ still has a reference from itself. Its reference count doesn't drop to zero.
Fortunately, Python's cyclic garbage collector will eventually figure out that
the list is garbage and free it.
-In the second version of the :class:`Custom` example, we allowed any kind of
-object to be stored in the :attr:`first` or :attr:`last` attributes [#]_.
+In the second version of the :class:`!Custom` example, we allowed any kind of
+object to be stored in the :attr:`!first` or :attr:`!last` attributes [#]_.
Besides, in the second and third versions, we allowed subclassing
-:class:`Custom`, and subclasses may add arbitrary attributes. For any of
-those two reasons, :class:`Custom` objects can participate in cycles:
+:class:`!Custom`, and subclasses may add arbitrary attributes. For any of
+those two reasons, :class:`!Custom` objects can participate in cycles:
.. code-block:: pycon
@@ -687,11 +678,11 @@ those two reasons, :class:`Custom` objects can participate in cycles:
>>> n = Derived()
>>> n.some_attribute = n
-To allow a :class:`Custom` instance participating in a reference cycle to
-be properly detected and collected by the cyclic GC, our :class:`Custom` type
+To allow a :class:`!Custom` instance participating in a reference cycle to
+be properly detected and collected by the cyclic GC, our :class:`!Custom` type
needs to fill two additional slots and to enable a flag that enables these slots:
-.. literalinclude:: ../includes/custom4.c
+.. literalinclude:: ../includes/newtypes/custom4.c
First, the traversal method lets the cyclic GC know about subobjects that could
@@ -715,8 +706,8 @@ participate in cycles::
}
For each subobject that can participate in cycles, we need to call the
-:c:func:`visit` function, which is passed to the traversal method. The
-:c:func:`visit` function takes as arguments the subobject and the extra argument
+:c:func:`!visit` function, which is passed to the traversal method. The
+:c:func:`!visit` function takes as arguments the subobject and the extra argument
*arg* passed to the traversal method. It returns an integer value that must be
returned if it is non-zero.
@@ -781,7 +772,7 @@ and ``Custom_clear``::
Py_TYPE(self)->tp_free((PyObject *) self);
}
-Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags::
+Finally, we add the :c:macro:`Py_TPFLAGS_HAVE_GC` flag to the class flags::
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
@@ -798,9 +789,9 @@ types. It is easiest to inherit from the built in types, since an extension can
easily use the :c:type:`PyTypeObject` it needs. It can be difficult to share
these :c:type:`PyTypeObject` structures between extension modules.
-In this example we will create a :class:`SubList` type that inherits from the
+In this example we will create a :class:`!SubList` type that inherits from the
built-in :class:`list` type. The new type will be completely compatible with
-regular lists, but will have an additional :meth:`increment` method that
+regular lists, but will have an additional :meth:`!increment` method that
increases an internal counter:
.. code-block:: pycon
@@ -815,10 +806,10 @@ increases an internal counter:
>>> print(s.increment())
2
-.. literalinclude:: ../includes/sublist.c
+.. literalinclude:: ../includes/newtypes/sublist.c
-As you can see, the source code closely resembles the :class:`Custom` examples in
+As you can see, the source code closely resembles the :class:`!Custom` examples in
previous sections. We will break down the main differences between them. ::
typedef struct {
@@ -830,7 +821,7 @@ The primary difference for derived type objects is that the base type's
object structure must be the first value. The base type will already include
the :c:func:`PyObject_HEAD` at the beginning of its structure.
-When a Python object is a :class:`SubList` instance, its ``PyObject *`` pointer
+When a Python object is a :class:`!SubList` instance, its ``PyObject *`` pointer
can be safely cast to both ``PyListObject *`` and ``SubListObject *``::
static int
@@ -842,7 +833,7 @@ can be safely cast to both ``PyListObject *`` and ``SubListObject *``::
return 0;
}
-We see above how to call through to the :attr:`__init__` method of the base
+We see above how to call through to the :meth:`~object.__init__` method of the base
type.
This pattern is important when writing a type with custom
@@ -869,9 +860,7 @@ function::
if (m == NULL)
return NULL;
- Py_INCREF(&SubListType);
- if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) {
- Py_DECREF(&SubListType);
+ if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) {
Py_DECREF(m);
return NULL;
}
@@ -886,7 +875,7 @@ slot with :c:func:`PyType_GenericNew` -- the allocation function from the base
type will be inherited.
After that, calling :c:func:`PyType_Ready` and adding the type object to the
-module is the same as with the basic :class:`Custom` examples.
+module is the same as with the basic :class:`!Custom` examples.
.. rubric:: Footnotes
diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst
index 1129b0968bc4e60..e366d6cb9f79e3b 100644
--- a/Doc/extending/windows.rst
+++ b/Doc/extending/windows.rst
@@ -132,4 +132,4 @@ modules (including Python) to be able to see your identifiers, you have to say
Developer Studio will throw in a lot of import libraries that you do not really
need, adding about 100K to your executable. To get rid of them, use the Project
Settings dialog, Link tab, to specify *ignore default libraries*. Add the
-correct :file:`msvcrtxx.lib` to the list of libraries.
+correct :file:`msvcrt{xx}.lib` to the list of libraries.
diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst
index 11d01374dc1e791..ae02c443e5938bf 100644
--- a/Doc/faq/design.rst
+++ b/Doc/faq/design.rst
@@ -584,9 +584,9 @@ exhaustive test suites that exercise every line of code in a module.
An appropriate testing discipline can help build large complex applications in
Python as well as having interface specifications would. In fact, it can be
better because an interface specification cannot test certain properties of a
-program. For example, the :meth:`list.append` method is expected to add new elements
+program. For example, the :meth:`!list.append` method is expected to add new elements
to the end of some internal list; an interface specification cannot test that
-your :meth:`list.append` implementation will actually do this correctly, but it's
+your :meth:`!list.append` implementation will actually do this correctly, but it's
trivial to check this property in a test suite.
Writing test suites is very helpful, and you might want to design your code to
diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst
index 07282639e4f9b4b..2a8b976925d042c 100644
--- a/Doc/faq/extending.rst
+++ b/Doc/faq/extending.rst
@@ -42,7 +42,7 @@ on what you're trying to do.
.. XXX make sure these all work
`Cython `_ and its relative `Pyrex
-`_ are compilers
+`_ are compilers
that accept a slightly modified form of Python and generate the corresponding
C code. Cython and Pyrex make it possible to write an extension without having
to learn Python's C API.
@@ -81,13 +81,13 @@ How do I extract C values from a Python object?
That depends on the object's type. If it's a tuple, :c:func:`PyTuple_Size`
returns its length and :c:func:`PyTuple_GetItem` returns the item at a specified
-index. Lists have similar functions, :c:func:`PyListSize` and
+index. Lists have similar functions, :c:func:`PyList_Size` and
:c:func:`PyList_GetItem`.
For bytes, :c:func:`PyBytes_Size` returns its length and
:c:func:`PyBytes_AsStringAndSize` provides a pointer to its value and its
length. Note that Python bytes objects may contain null bytes so C's
-:c:func:`strlen` should not be used.
+:c:func:`!strlen` should not be used.
To test the type of an object, first make sure it isn't ``NULL``, and then use
:c:func:`PyBytes_Check`, :c:func:`PyTuple_Check`, :c:func:`PyList_Check`, etc.
diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst
index 489bca76432d859..8727332594bda69 100644
--- a/Doc/faq/general.rst
+++ b/Doc/faq/general.rst
@@ -54,8 +54,8 @@ commercial use, to sell copies of Python in source or binary form (modified or
unmodified), or to sell products that incorporate Python in some form. We would
still like to know about all commercial use of Python, of course.
-See `the PSF license page `_ to find further
-explanations and a link to the full text of the license.
+See `the license page `_ to find further
+explanations and the full text of the PSF License.
The Python logo is trademarked, and in certain cases permission is required to
use it. Consult `the Trademark Usage Policy
@@ -135,7 +135,7 @@ Python versions are numbered "A.B.C" or "A.B":
See :pep:`6` for more information about bugfix releases.
-Not all releases are bugfix releases. In the run-up to a new major release, a
+Not all releases are bugfix releases. In the run-up to a new feature release, a
series of development releases are made, denoted as alpha, beta, or release
candidate. Alphas are early releases in which interfaces aren't yet finalized;
it's not unexpected to see an interface change between two alpha releases.
@@ -215,7 +215,7 @@ every day, and Usenet readers are often more able to cope with this volume.
Announcements of new software releases and events can be found in
comp.lang.python.announce, a low-traffic moderated list that receives about five
postings per day. It's available as `the python-announce mailing list
-`_.
+`_.
More info about other mailing lists and newsgroups
can be found at https://www.python.org/community/lists/.
@@ -248,8 +248,8 @@ Are there any published articles about Python that I can reference?
It's probably best to cite your favorite book about Python.
-The very first article about Python was written in 1991 and is now quite
-outdated.
+The `very first article `_ about Python was
+written in 1991 and is now quite outdated.
Guido van Rossum and Jelke de Boer, "Interactively Testing Remote Servers
Using the Python Programming Language", CWI Quarterly, Volume 4, Issue 4
@@ -297,9 +297,9 @@ How stable is Python?
Very stable. New, stable releases have been coming out roughly every 6 to 18
months since 1991, and this seems likely to continue. As of version 3.9,
-Python will have a major new release every 12 months (:pep:`602`).
+Python will have a new feature release every 12 months (:pep:`602`).
-The developers issue "bugfix" releases of older versions, so the stability of
+The developers issue bugfix releases of older versions, so the stability of
existing releases gradually improves. Bugfix releases, indicated by a third
component of the version number (e.g. 3.5.3, 3.6.2), are managed for stability;
only fixes for known problems are included in a bugfix release, and it's
@@ -352,7 +352,7 @@ titled "Python X.Y Release Schedule", where X.Y is a version that hasn't been
publicly released yet.
New development is discussed on `the python-dev mailing list
-`_.
+`_.
Is it reasonable to propose incompatible changes to Python?
diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst
index 023ffdf0db510a6..cfa60feceb31b77 100644
--- a/Doc/faq/gui.rst
+++ b/Doc/faq/gui.rst
@@ -43,18 +43,11 @@ applications, the applications will not be truly stand-alone, as the application
will still need the Tcl and Tk libraries.
One solution is to ship the application with the Tcl and Tk libraries, and point
-to them at run-time using the :envvar:`TCL_LIBRARY` and :envvar:`TK_LIBRARY`
+to them at run-time using the :envvar:`!TCL_LIBRARY` and :envvar:`!TK_LIBRARY`
environment variables.
-To get truly stand-alone applications, the Tcl scripts that form the library
-have to be integrated into the application as well. One tool supporting that is
-SAM (stand-alone modules), which is part of the Tix distribution
-(https://tix.sourceforge.net/).
-
-Build Tix with SAM enabled, perform the appropriate call to
-:c:func:`Tclsam_init`, etc. inside Python's
-:file:`Modules/tkappinit.c`, and link with libtclsam and libtksam (you
-might include the Tix libraries as well).
+Various third-party freeze libraries such as py2exe and cx_Freeze have
+handling for Tkinter applications built-in.
Can I have Tk events handled while waiting for I/O?
@@ -62,7 +55,7 @@ Can I have Tk events handled while waiting for I/O?
On platforms other than Windows, yes, and you don't even
need threads! But you'll have to restructure your I/O
-code a bit. Tk has the equivalent of Xt's :c:func:`XtAddInput()` call, which allows you
+code a bit. Tk has the equivalent of Xt's :c:func:`!XtAddInput` call, which allows you
to register a callback function which will be called from the Tk mainloop when
I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`.
@@ -70,8 +63,9 @@ I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`.
I can't get key bindings to work in Tkinter: why?
-------------------------------------------------
-An often-heard complaint is that event handlers bound to events with the
-:meth:`bind` method don't get handled even when the appropriate key is pressed.
+An often-heard complaint is that event handlers :ref:`bound `
+to events with the :meth:`!bind` method
+don't get handled even when the appropriate key is pressed.
The most common cause is that the widget to which the binding applies doesn't
have "keyboard focus". Check out the Tk documentation for the focus command.
diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst
index a9cde456575020f..476a43d9c288f18 100644
--- a/Doc/faq/library.rst
+++ b/Doc/faq/library.rst
@@ -111,7 +111,7 @@ Is there an equivalent to C's onexit() in Python?
-------------------------------------------------
The :mod:`atexit` module provides a register function that is similar to C's
-:c:func:`onexit`.
+:c:func:`!onexit`.
Why don't my signal handlers work?
@@ -397,7 +397,7 @@ These aren't::
D[x] = D[x] + 1
Operations that replace other objects may invoke those other objects'
-:meth:`__del__` method when their reference count reaches zero, and that can
+:meth:`~object.__del__` method when their reference count reaches zero, and that can
affect things. This is especially true for the mass updates to dictionaries and
lists. When in doubt, use a mutex!
@@ -566,7 +566,7 @@ use ``p.read(n)``.
Note on a bug in popen2: unless your program calls ``wait()`` or
``waitpid()``, finished child processes are never removed, and eventually
calls to popen2 will fail because of a limit on the number of child
- processes. Calling :func:`os.waitpid` with the :data:`os.WNOHANG` option can
+ processes. Calling :func:`os.waitpid` with the :const:`os.WNOHANG` option can
prevent this; a good place to insert such a call would be before calling
``popen2`` again.
@@ -669,41 +669,6 @@ and client-side web systems.
A summary of available frameworks is maintained by Paul Boddie at
https://wiki.python.org/moin/WebProgramming\ .
-Cameron Laird maintains a useful set of pages about Python web technologies at
-https://web.archive.org/web/20210224183619/http://phaseit.net/claird/comp.lang.python/web_python.
-
-
-How can I mimic CGI form submission (METHOD=POST)?
---------------------------------------------------
-
-I would like to retrieve web pages that are the result of POSTing a form. Is
-there existing code that would let me do this easily?
-
-Yes. Here's a simple example that uses :mod:`urllib.request`::
-
- #!/usr/local/bin/python
-
- import urllib.request
-
- # build the query string
- qs = "First=Josephine&MI=Q&Last=Public"
-
- # connect and send the server a path
- req = urllib.request.urlopen('http://www.some-server.out-there'
- '/cgi-bin/some-cgi-script', data=qs)
- with req:
- msg, hdrs = req.read(), req.info()
-
-Note that in general for percent-encoded POST operations, query strings must be
-quoted using :func:`urllib.parse.urlencode`. For example, to send
-``name=Guy Steele, Jr.``::
-
- >>> import urllib.parse
- >>> urllib.parse.urlencode({'name': 'Guy Steele, Jr.'})
- 'name=Guy+Steele%2C+Jr.'
-
-.. seealso:: :ref:`urllib-howto` for extensive examples.
-
What module should I use to help with generating HTML?
------------------------------------------------------
@@ -765,14 +730,17 @@ The :mod:`select` module is commonly used to help with asynchronous I/O on
sockets.
To prevent the TCP connect from blocking, you can set the socket to non-blocking
-mode. Then when you do the :meth:`socket.connect`, you will either connect immediately
+mode. Then when you do the :meth:`~socket.socket.connect`,
+you will either connect immediately
(unlikely) or get an exception that contains the error number as ``.errno``.
``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't
finished yet. Different OSes will return different values, so you're going to
have to check what's returned on your system.
-You can use the :meth:`socket.connect_ex` method to avoid creating an exception. It will
-just return the errno value. To poll, you can call :meth:`socket.connect_ex` again later
+You can use the :meth:`~socket.socket.connect_ex` method
+to avoid creating an exception.
+It will just return the errno value.
+To poll, you can call :meth:`~socket.socket.connect_ex` again later
-- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this
socket to :meth:`select.select` to check if it's writable.
@@ -780,7 +748,7 @@ socket to :meth:`select.select` to check if it's writable.
The :mod:`asyncio` module provides a general purpose single-threaded and
concurrent asynchronous library, which can be used for writing non-blocking
network code.
- The third-party `Twisted `_ library is
+ The third-party `Twisted `_ library is
a popular and feature-rich alternative.
diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst
index f3c5b0f76494aae..0a88c5f6384f2be 100644
--- a/Doc/faq/programming.rst
+++ b/Doc/faq/programming.rst
@@ -61,7 +61,7 @@ Yes.
`Pyflakes `_ do basic checking that will
help you catch bugs sooner.
-Static type checkers such as `Mypy `_,
+Static type checkers such as `Mypy `_,
`Pyre `_, and
`Pytype `_ can check type hints in Python
source code.
@@ -113,6 +113,8 @@ Yes. The coding style required for standard library modules is documented as
Core Language
=============
+.. _faq-unboundlocalerror:
+
Why am I getting an UnboundLocalError when the variable has a value?
--------------------------------------------------------------------
@@ -452,7 +454,7 @@ There are two factors that produce this result:
(the list), and both ``x`` and ``y`` refer to it.
2) Lists are :term:`mutable`, which means that you can change their content.
-After the call to :meth:`~list.append`, the content of the mutable object has
+After the call to :meth:`!append`, the content of the mutable object has
changed from ``[]`` to ``[10]``. Since both the variables refer to the same
object, using either name accesses the modified value ``[10]``.
@@ -922,12 +924,12 @@ module::
'Hello, there!'
>>> import array
- >>> a = array.array('u', s)
+ >>> a = array.array('w', s)
>>> print(a)
- array('u', 'Hello, world')
+ array('w', 'Hello, world')
>>> a[0] = 'y'
>>> print(a)
- array('u', 'yello, world')
+ array('w', 'yello, world')
>>> a.tounicode()
'yello, world'
@@ -1024,6 +1026,46 @@ What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean?
See the :ref:`unicode-howto`.
+.. _faq-programming-raw-string-backslash:
+
+Can I end a raw string with an odd number of backslashes?
+---------------------------------------------------------
+
+A raw string ending with an odd number of backslashes will escape the string's quote::
+
+ >>> r'C:\this\will\not\work\'
+ File "", line 1
+ r'C:\this\will\not\work\'
+ ^
+ SyntaxError: unterminated string literal (detected at line 1)
+
+There are several workarounds for this. One is to use regular strings and double
+the backslashes::
+
+ >>> 'C:\\this\\will\\work\\'
+ 'C:\\this\\will\\work\\'
+
+Another is to concatenate a regular string containing an escaped backslash to the
+raw string::
+
+ >>> r'C:\this\will\work' '\\'
+ 'C:\\this\\will\\work\\'
+
+It is also possible to use :func:`os.path.join` to append a backslash on Windows::
+
+ >>> os.path.join(r'C:\this\will\work', '')
+ 'C:\\this\\will\\work\\'
+
+Note that while a backslash will "escape" a quote for the purposes of
+determining where the raw string ends, no escaping occurs when interpreting the
+value of the raw string. That is, the backslash remains present in the value of
+the raw string::
+
+ >>> r'backslash\'preserved'
+ "backslash\\'preserved"
+
+Also see the specification in the :ref:`language reference `.
+
Performance
===========
@@ -1279,13 +1321,25 @@ Or, you can use an extension that provides a matrix datatype; `NumPy
`_ is the best known.
-How do I apply a method to a sequence of objects?
--------------------------------------------------
+How do I apply a method or function to a sequence of objects?
+-------------------------------------------------------------
-Use a list comprehension::
+To call a method or function and accumulate the return values is a list,
+a :term:`list comprehension` is an elegant solution::
result = [obj.method() for obj in mylist]
+ result = [function(obj) for obj in mylist]
+
+To just run the method or function without saving the return values,
+a plain :keyword:`for` loop will suffice::
+
+ for obj in mylist:
+ obj.method()
+
+ for obj in mylist:
+ function(obj)
+
.. _faq-augmented-assignment-tuple-error:
Why does a_tuple[i] += ['item'] raise an exception when the addition works?
@@ -1343,7 +1397,7 @@ To see why this happens, you need to know that (a) if an object implements an
:meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented
assignment
is executed, and its return value is what gets used in the assignment statement;
-and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`~list.extend` on the list
+and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list
and returning the list. That's why we say that for lists, ``+=`` is a
"shorthand" for :meth:`!list.extend`::
@@ -1849,7 +1903,7 @@ identity tests. This prevents the code from being confused by objects such as
``float('NaN')`` that are not equal to themselves.
For example, here is the implementation of
-:meth:`collections.abc.Sequence.__contains__`::
+:meth:`!collections.abc.Sequence.__contains__`::
def __contains__(self, value):
for v in self:
@@ -1925,7 +1979,7 @@ method result will be released right away. The disadvantage is that if
instances accumulate, so too will the accumulated method results. They
can grow without bound.
-The *lru_cache* approach works with methods that have hashable
+The *lru_cache* approach works with methods that have :term:`hashable`
arguments. It creates a reference to the instance unless special
efforts are made to pass in weak references.
diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst
index 7768aafd979685b..c0c92fdbbc84d12 100644
--- a/Doc/faq/windows.rst
+++ b/Doc/faq/windows.rst
@@ -167,7 +167,7 @@ How can I embed Python into a Windows application?
Embedding the Python interpreter in a Windows app can be summarized as follows:
-1. Do _not_ build Python into your .exe file directly. On Windows, Python must
+1. Do **not** build Python into your .exe file directly. On Windows, Python must
be a DLL to handle importing modules that are themselves DLL's. (This is the
first key undocumented fact.) Instead, link to :file:`python{NN}.dll`; it is
typically installed in ``C:\Windows\System``. *NN* is the Python version, a
@@ -191,7 +191,7 @@ Embedding the Python interpreter in a Windows app can be summarized as follows:
2. If you use SWIG, it is easy to create a Python "extension module" that will
make the app's data and methods available to Python. SWIG will handle just
about all the grungy details for you. The result is C code that you link
- *into* your .exe file (!) You do _not_ have to create a DLL file, and this
+ *into* your .exe file (!) You do **not** have to create a DLL file, and this
also simplifies linking.
3. SWIG will create an init function (a C function) whose name depends on the
@@ -218,10 +218,10 @@ Embedding the Python interpreter in a Windows app can be summarized as follows:
5. There are two problems with Python's C API which will become apparent if you
use a compiler other than MSVC, the compiler used to build pythonNN.dll.
- Problem 1: The so-called "Very High Level" functions that take FILE *
+ Problem 1: The so-called "Very High Level" functions that take ``FILE *``
arguments will not work in a multi-compiler environment because each
- compiler's notion of a struct FILE will be different. From an implementation
- standpoint these are very _low_ level functions.
+ compiler's notion of a ``struct FILE`` will be different. From an implementation
+ standpoint these are very low level functions.
Problem 2: SWIG generates the following code when generating wrappers to void
functions:
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 3d74d550dc345a5..6b517b95f970138 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -22,15 +22,6 @@ Glossary
* The :const:`Ellipsis` built-in constant.
- 2to3
- A tool that tries to convert Python 2.x code to Python 3.x code by
- handling most of the incompatibilities which can be detected by parsing the
- source and traversing the parse tree.
-
- 2to3 is available in the standard library as :mod:`lib2to3`; a standalone
- entry point is provided as :file:`Tools/scripts/2to3`. See
- :ref:`2to3-reference`.
-
abstract base class
Abstract base classes complement :term:`duck-typing` by
providing a way to define interfaces when other techniques like
@@ -92,8 +83,8 @@ Glossary
asynchronous context manager
An object which controls the environment seen in an
- :keyword:`async with` statement by defining :meth:`__aenter__` and
- :meth:`__aexit__` methods. Introduced by :pep:`492`.
+ :keyword:`async with` statement by defining :meth:`~object.__aenter__` and
+ :meth:`~object.__aexit__` methods. Introduced by :pep:`492`.
asynchronous generator
A function which returns an :term:`asynchronous generator iterator`. It
@@ -113,26 +104,26 @@ Glossary
An object created by a :term:`asynchronous generator` function.
This is an :term:`asynchronous iterator` which when called using the
- :meth:`__anext__` method returns an awaitable object which will execute
+ :meth:`~object.__anext__` method returns an awaitable object which will execute
the body of the asynchronous generator function until the next
:keyword:`yield` expression.
Each :keyword:`yield` temporarily suspends processing, remembering the
location execution state (including local variables and pending
try-statements). When the *asynchronous generator iterator* effectively
- resumes with another awaitable returned by :meth:`__anext__`, it
+ resumes with another awaitable returned by :meth:`~object.__anext__`, it
picks up where it left off. See :pep:`492` and :pep:`525`.
asynchronous iterable
An object, that can be used in an :keyword:`async for` statement.
Must return an :term:`asynchronous iterator` from its
- :meth:`__aiter__` method. Introduced by :pep:`492`.
+ :meth:`~object.__aiter__` method. Introduced by :pep:`492`.
asynchronous iterator
- An object that implements the :meth:`__aiter__` and :meth:`__anext__`
- methods. ``__anext__`` must return an :term:`awaitable` object.
+ An object that implements the :meth:`~object.__aiter__` and :meth:`~object.__anext__`
+ methods. :meth:`~object.__anext__` must return an :term:`awaitable` object.
:keyword:`async for` resolves the awaitables returned by an asynchronous
- iterator's :meth:`__anext__` method until it raises a
+ iterator's :meth:`~object.__anext__` method until it raises a
:exc:`StopAsyncIteration` exception. Introduced by :pep:`492`.
attribute
@@ -149,7 +140,7 @@ Glossary
awaitable
An object that can be used in an :keyword:`await` expression. Can be
- a :term:`coroutine` or an object with an :meth:`__await__` method.
+ a :term:`coroutine` or an object with an :meth:`~object.__await__` method.
See also :pep:`492`.
BDFL
@@ -168,8 +159,9 @@ Glossary
:class:`str` objects.
borrowed reference
- In Python's C API, a borrowed reference is a reference to an object.
- It does not modify the object reference count. It becomes a dangling
+ In Python's C API, a borrowed reference is a reference to an object,
+ where the code using the object does not own the reference.
+ It becomes a dangling
pointer if the object is destroyed. For example, a garbage collection can
remove the last :term:`strong reference` to the object and so destroy it.
@@ -214,7 +206,7 @@ Glossary
A callable is an object that can be called, possibly with a set
of arguments (see :term:`argument`), with the following syntax::
- callable(argument1, argument2, ...)
+ callable(argument1, argument2, argumentN)
A :term:`function`, and by extension a :term:`method`, is a callable.
An instance of a class that implements the :meth:`~object.__call__`
@@ -247,7 +239,7 @@ Glossary
context manager
An object which controls the environment seen in a :keyword:`with`
- statement by defining :meth:`__enter__` and :meth:`__exit__` methods.
+ statement by defining :meth:`~object.__enter__` and :meth:`~object.__exit__` methods.
See :pep:`343`.
context variable
@@ -587,6 +579,16 @@ Glossary
:ref:`idle` is a basic editor and interpreter environment
which ships with the standard distribution of Python.
+ immortal
+ If an object is immortal, its reference count is never modified, and
+ therefore it is never deallocated.
+
+ Built-in strings and singletons are immortal objects. For example,
+ :const:`True` and :const:`None` singletons are immmortal.
+
+ See `PEP 683 – Immortal Objects, Using a Fixed Refcount
+ `_ for more information.
+
immutable
An object with a fixed value. Immutable objects include numbers, strings and
tuples. Such an object cannot be altered. A new object has to
@@ -644,7 +646,7 @@ Glossary
iterables include all sequence types (such as :class:`list`, :class:`str`,
and :class:`tuple`) and some non-sequence types like :class:`dict`,
:term:`file objects `, and objects of any classes you define
- with an :meth:`__iter__` method or with a :meth:`__getitem__` method
+ with an :meth:`__iter__` method or with a :meth:`~object.__getitem__` method
that implements :term:`sequence` semantics.
Iterables can be
@@ -1063,7 +1065,9 @@ Glossary
reference count
The number of references to an object. When the reference count of an
- object drops to zero, it is deallocated. Reference counting is
+ object drops to zero, it is deallocated. Some objects are
+ :term:`immortal` and have reference counts that are never modified, and
+ therefore the objects are never deallocated. Reference counting is
generally not visible to Python code, but it is a key element of the
:term:`CPython` implementation. Programmers can call the
:func:`sys.getrefcount` function to return the
@@ -1084,17 +1088,17 @@ Glossary
sequence
An :term:`iterable` which supports efficient element access using integer
- indices via the :meth:`__getitem__` special method and defines a
+ indices via the :meth:`~object.__getitem__` special method and defines a
:meth:`__len__` method that returns the length of the sequence.
Some built-in sequence types are :class:`list`, :class:`str`,
:class:`tuple`, and :class:`bytes`. Note that :class:`dict` also
- supports :meth:`__getitem__` and :meth:`__len__`, but is considered a
+ supports :meth:`~object.__getitem__` and :meth:`__len__`, but is considered a
mapping rather than a sequence because the lookups use arbitrary
:term:`immutable` keys rather than integers.
The :class:`collections.abc.Sequence` abstract base class
defines a much richer interface that goes beyond just
- :meth:`__getitem__` and :meth:`__len__`, adding :meth:`count`,
+ :meth:`~object.__getitem__` and :meth:`__len__`, adding :meth:`count`,
:meth:`index`, :meth:`__contains__`, and
:meth:`__reversed__`. Types that implement this expanded
interface can be registered explicitly using
@@ -1116,6 +1120,21 @@ Glossary
when several are given, such as in ``variable_name[1:3:5]``. The bracket
(subscript) notation uses :class:`slice` objects internally.
+ soft deprecated
+ A soft deprecation can be used when using an API which should no longer
+ be used to write new code, but it remains safe to continue using it in
+ existing code. The API remains documented and tested, but will not be
+ developed further (no enhancement).
+
+ The main difference between a "soft" and a (regular) "hard" deprecation
+ is that the soft deprecation does not imply scheduling the removal of the
+ deprecated API.
+
+ Another difference is that a soft deprecation does not issue a warning.
+
+ See `PEP 387: Soft Deprecation
+ `_.
+
special method
.. index:: pair: special; method
@@ -1129,10 +1148,17 @@ Glossary
an :term:`expression` or one of several constructs with a keyword, such
as :keyword:`if`, :keyword:`while` or :keyword:`for`.
+ static type checker
+ An external tool that reads Python code and analyzes it, looking for
+ issues such as incorrect types. See also :term:`type hints `
+ and the :mod:`typing` module.
+
strong reference
In Python's C API, a strong reference is a reference to an object
- which increments the object's reference count when it is created and
- decrements the object's reference count when it is deleted.
+ which is owned by the code holding the reference. The strong
+ reference is taken by calling :c:func:`Py_INCREF` when the
+ reference is created and released with :c:func:`Py_DECREF`
+ when the reference is deleted.
The :c:func:`Py_NewRef` function can be used to create a strong reference
to an object. Usually, the :c:func:`Py_DECREF` function must be called on
@@ -1203,8 +1229,8 @@ Glossary
attribute, or a function parameter or return value.
Type hints are optional and are not enforced by Python but
- they are useful to static type analysis tools, and aid IDEs with code
- completion and refactoring.
+ they are useful to :term:`static type checkers `.
+ They can also aid IDEs with code completion and refactoring.
Type hints of global variables, class attributes, and functions,
but not local variables, can be accessed using
diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst
index 2bc2f2d4c839e2d..1134686c947d668 100644
--- a/Doc/howto/annotations.rst
+++ b/Doc/howto/annotations.rst
@@ -32,195 +32,201 @@ Annotations Best Practices
Accessing The Annotations Dict Of An Object In Python 3.10 And Newer
====================================================================
- Python 3.10 adds a new function to the standard library:
- :func:`inspect.get_annotations`. In Python versions 3.10
- and newer, calling this function is the best practice for
- accessing the annotations dict of any object that supports
- annotations. This function can also "un-stringize"
- stringized annotations for you.
-
- If for some reason :func:`inspect.get_annotations` isn't
- viable for your use case, you may access the
- ``__annotations__`` data member manually. Best practice
- for this changed in Python 3.10 as well: as of Python 3.10,
- ``o.__annotations__`` is guaranteed to *always* work
- on Python functions, classes, and modules. If you're
- certain the object you're examining is one of these three
- *specific* objects, you may simply use ``o.__annotations__``
- to get at the object's annotations dict.
-
- However, other types of callables--for example,
- callables created by :func:`functools.partial`--may
- not have an ``__annotations__`` attribute defined. When
- accessing the ``__annotations__`` of a possibly unknown
- object, best practice in Python versions 3.10 and
- newer is to call :func:`getattr` with three arguments,
- for example ``getattr(o, '__annotations__', None)``.
+Python 3.10 adds a new function to the standard library:
+:func:`inspect.get_annotations`. In Python versions 3.10
+and newer, calling this function is the best practice for
+accessing the annotations dict of any object that supports
+annotations. This function can also "un-stringize"
+stringized annotations for you.
+
+If for some reason :func:`inspect.get_annotations` isn't
+viable for your use case, you may access the
+``__annotations__`` data member manually. Best practice
+for this changed in Python 3.10 as well: as of Python 3.10,
+``o.__annotations__`` is guaranteed to *always* work
+on Python functions, classes, and modules. If you're
+certain the object you're examining is one of these three
+*specific* objects, you may simply use ``o.__annotations__``
+to get at the object's annotations dict.
+
+However, other types of callables--for example,
+callables created by :func:`functools.partial`--may
+not have an ``__annotations__`` attribute defined. When
+accessing the ``__annotations__`` of a possibly unknown
+object, best practice in Python versions 3.10 and
+newer is to call :func:`getattr` with three arguments,
+for example ``getattr(o, '__annotations__', None)``.
+
+Before Python 3.10, accessing ``__annotations__`` on a class that
+defines no annotations but that has a parent class with
+annotations would return the parent's ``__annotations__``.
+In Python 3.10 and newer, the child class's annotations
+will be an empty dict instead.
Accessing The Annotations Dict Of An Object In Python 3.9 And Older
===================================================================
- In Python 3.9 and older, accessing the annotations dict
- of an object is much more complicated than in newer versions.
- The problem is a design flaw in these older versions of Python,
- specifically to do with class annotations.
+In Python 3.9 and older, accessing the annotations dict
+of an object is much more complicated than in newer versions.
+The problem is a design flaw in these older versions of Python,
+specifically to do with class annotations.
- Best practice for accessing the annotations dict of other
- objects--functions, other callables, and modules--is the same
- as best practice for 3.10, assuming you aren't calling
- :func:`inspect.get_annotations`: you should use three-argument
- :func:`getattr` to access the object's ``__annotations__``
- attribute.
+Best practice for accessing the annotations dict of other
+objects--functions, other callables, and modules--is the same
+as best practice for 3.10, assuming you aren't calling
+:func:`inspect.get_annotations`: you should use three-argument
+:func:`getattr` to access the object's ``__annotations__``
+attribute.
- Unfortunately, this isn't best practice for classes. The problem
- is that, since ``__annotations__`` is optional on classes, and
- because classes can inherit attributes from their base classes,
- accessing the ``__annotations__`` attribute of a class may
- inadvertently return the annotations dict of a *base class.*
- As an example::
+Unfortunately, this isn't best practice for classes. The problem
+is that, since ``__annotations__`` is optional on classes, and
+because classes can inherit attributes from their base classes,
+accessing the ``__annotations__`` attribute of a class may
+inadvertently return the annotations dict of a *base class.*
+As an example::
- class Base:
- a: int = 3
- b: str = 'abc'
+ class Base:
+ a: int = 3
+ b: str = 'abc'
- class Derived(Base):
- pass
+ class Derived(Base):
+ pass
- print(Derived.__annotations__)
+ print(Derived.__annotations__)
- This will print the annotations dict from ``Base``, not
- ``Derived``.
+This will print the annotations dict from ``Base``, not
+``Derived``.
- Your code will have to have a separate code path if the object
- you're examining is a class (``isinstance(o, type)``).
- In that case, best practice relies on an implementation detail
- of Python 3.9 and before: if a class has annotations defined,
- they are stored in the class's ``__dict__`` dictionary. Since
- the class may or may not have annotations defined, best practice
- is to call the ``get`` method on the class dict.
+Your code will have to have a separate code path if the object
+you're examining is a class (``isinstance(o, type)``).
+In that case, best practice relies on an implementation detail
+of Python 3.9 and before: if a class has annotations defined,
+they are stored in the class's ``__dict__`` dictionary. Since
+the class may or may not have annotations defined, best practice
+is to call the ``get`` method on the class dict.
- To put it all together, here is some sample code that safely
- accesses the ``__annotations__`` attribute on an arbitrary
- object in Python 3.9 and before::
+To put it all together, here is some sample code that safely
+accesses the ``__annotations__`` attribute on an arbitrary
+object in Python 3.9 and before::
- if isinstance(o, type):
- ann = o.__dict__.get('__annotations__', None)
- else:
- ann = getattr(o, '__annotations__', None)
+ if isinstance(o, type):
+ ann = o.__dict__.get('__annotations__', None)
+ else:
+ ann = getattr(o, '__annotations__', None)
- After running this code, ``ann`` should be either a
- dictionary or ``None``. You're encouraged to double-check
- the type of ``ann`` using :func:`isinstance` before further
- examination.
+After running this code, ``ann`` should be either a
+dictionary or ``None``. You're encouraged to double-check
+the type of ``ann`` using :func:`isinstance` before further
+examination.
- Note that some exotic or malformed type objects may not have
- a ``__dict__`` attribute, so for extra safety you may also wish
- to use :func:`getattr` to access ``__dict__``.
+Note that some exotic or malformed type objects may not have
+a ``__dict__`` attribute, so for extra safety you may also wish
+to use :func:`getattr` to access ``__dict__``.
Manually Un-Stringizing Stringized Annotations
==============================================
- In situations where some annotations may be "stringized",
- and you wish to evaluate those strings to produce the
- Python values they represent, it really is best to
- call :func:`inspect.get_annotations` to do this work
- for you.
-
- If you're using Python 3.9 or older, or if for some reason
- you can't use :func:`inspect.get_annotations`, you'll need
- to duplicate its logic. You're encouraged to examine the
- implementation of :func:`inspect.get_annotations` in the
- current Python version and follow a similar approach.
-
- In a nutshell, if you wish to evaluate a stringized annotation
- on an arbitrary object ``o``:
-
- * If ``o`` is a module, use ``o.__dict__`` as the
- ``globals`` when calling :func:`eval`.
- * If ``o`` is a class, use ``sys.modules[o.__module__].__dict__``
- as the ``globals``, and ``dict(vars(o))`` as the ``locals``,
- when calling :func:`eval`.
- * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`,
- :func:`functools.wraps`, or :func:`functools.partial`, iteratively
- unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as
- appropriate, until you have found the root unwrapped function.
- * If ``o`` is a callable (but not a class), use
- ``o.__globals__`` as the globals when calling :func:`eval`.
-
- However, not all string values used as annotations can
- be successfully turned into Python values by :func:`eval`.
- String values could theoretically contain any valid string,
- and in practice there are valid use cases for type hints that
- require annotating with string values that specifically
- *can't* be evaluated. For example:
-
- * :pep:`604` union types using ``|``, before support for this
- was added to Python 3.10.
- * Definitions that aren't needed at runtime, only imported
- when :const:`typing.TYPE_CHECKING` is true.
-
- If :func:`eval` attempts to evaluate such values, it will
- fail and raise an exception. So, when designing a library
- API that works with annotations, it's recommended to only
- attempt to evaluate string values when explicitly requested
- to by the caller.
+In situations where some annotations may be "stringized",
+and you wish to evaluate those strings to produce the
+Python values they represent, it really is best to
+call :func:`inspect.get_annotations` to do this work
+for you.
+
+If you're using Python 3.9 or older, or if for some reason
+you can't use :func:`inspect.get_annotations`, you'll need
+to duplicate its logic. You're encouraged to examine the
+implementation of :func:`inspect.get_annotations` in the
+current Python version and follow a similar approach.
+
+In a nutshell, if you wish to evaluate a stringized annotation
+on an arbitrary object ``o``:
+
+* If ``o`` is a module, use ``o.__dict__`` as the
+ ``globals`` when calling :func:`eval`.
+* If ``o`` is a class, use ``sys.modules[o.__module__].__dict__``
+ as the ``globals``, and ``dict(vars(o))`` as the ``locals``,
+ when calling :func:`eval`.
+* If ``o`` is a wrapped callable using :func:`functools.update_wrapper`,
+ :func:`functools.wraps`, or :func:`functools.partial`, iteratively
+ unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as
+ appropriate, until you have found the root unwrapped function.
+* If ``o`` is a callable (but not a class), use
+ ``o.__globals__`` as the globals when calling :func:`eval`.
+
+However, not all string values used as annotations can
+be successfully turned into Python values by :func:`eval`.
+String values could theoretically contain any valid string,
+and in practice there are valid use cases for type hints that
+require annotating with string values that specifically
+*can't* be evaluated. For example:
+
+* :pep:`604` union types using ``|``, before support for this
+ was added to Python 3.10.
+* Definitions that aren't needed at runtime, only imported
+ when :const:`typing.TYPE_CHECKING` is true.
+
+If :func:`eval` attempts to evaluate such values, it will
+fail and raise an exception. So, when designing a library
+API that works with annotations, it's recommended to only
+attempt to evaluate string values when explicitly requested
+to by the caller.
Best Practices For ``__annotations__`` In Any Python Version
============================================================
- * You should avoid assigning to the ``__annotations__`` member
- of objects directly. Let Python manage setting ``__annotations__``.
+* You should avoid assigning to the ``__annotations__`` member
+ of objects directly. Let Python manage setting ``__annotations__``.
- * If you do assign directly to the ``__annotations__`` member
- of an object, you should always set it to a ``dict`` object.
+* If you do assign directly to the ``__annotations__`` member
+ of an object, you should always set it to a ``dict`` object.
- * If you directly access the ``__annotations__`` member
- of an object, you should ensure that it's a
- dictionary before attempting to examine its contents.
+* If you directly access the ``__annotations__`` member
+ of an object, you should ensure that it's a
+ dictionary before attempting to examine its contents.
- * You should avoid modifying ``__annotations__`` dicts.
+* You should avoid modifying ``__annotations__`` dicts.
- * You should avoid deleting the ``__annotations__`` attribute
- of an object.
+* You should avoid deleting the ``__annotations__`` attribute
+ of an object.
``__annotations__`` Quirks
==========================
- In all versions of Python 3, function
- objects lazy-create an annotations dict if no annotations
- are defined on that object. You can delete the ``__annotations__``
- attribute using ``del fn.__annotations__``, but if you then
- access ``fn.__annotations__`` the object will create a new empty dict
- that it will store and return as its annotations. Deleting the
- annotations on a function before it has lazily created its annotations
- dict will throw an ``AttributeError``; using ``del fn.__annotations__``
- twice in a row is guaranteed to always throw an ``AttributeError``.
-
- Everything in the above paragraph also applies to class and module
- objects in Python 3.10 and newer.
-
- In all versions of Python 3, you can set ``__annotations__``
- on a function object to ``None``. However, subsequently
- accessing the annotations on that object using ``fn.__annotations__``
- will lazy-create an empty dictionary as per the first paragraph of
- this section. This is *not* true of modules and classes, in any Python
- version; those objects permit setting ``__annotations__`` to any
- Python value, and will retain whatever value is set.
-
- If Python stringizes your annotations for you
- (using ``from __future__ import annotations``), and you
- specify a string as an annotation, the string will
- itself be quoted. In effect the annotation is quoted
- *twice.* For example::
-
- from __future__ import annotations
- def foo(a: "str"): pass
-
- print(foo.__annotations__)
-
- This prints ``{'a': "'str'"}``. This shouldn't really be considered
- a "quirk"; it's mentioned here simply because it might be surprising.
+In all versions of Python 3, function
+objects lazy-create an annotations dict if no annotations
+are defined on that object. You can delete the ``__annotations__``
+attribute using ``del fn.__annotations__``, but if you then
+access ``fn.__annotations__`` the object will create a new empty dict
+that it will store and return as its annotations. Deleting the
+annotations on a function before it has lazily created its annotations
+dict will throw an ``AttributeError``; using ``del fn.__annotations__``
+twice in a row is guaranteed to always throw an ``AttributeError``.
+
+Everything in the above paragraph also applies to class and module
+objects in Python 3.10 and newer.
+
+In all versions of Python 3, you can set ``__annotations__``
+on a function object to ``None``. However, subsequently
+accessing the annotations on that object using ``fn.__annotations__``
+will lazy-create an empty dictionary as per the first paragraph of
+this section. This is *not* true of modules and classes, in any Python
+version; those objects permit setting ``__annotations__`` to any
+Python value, and will retain whatever value is set.
+
+If Python stringizes your annotations for you
+(using ``from __future__ import annotations``), and you
+specify a string as an annotation, the string will
+itself be quoted. In effect the annotation is quoted
+*twice.* For example::
+
+ from __future__ import annotations
+ def foo(a: "str"): pass
+
+ print(foo.__annotations__)
+
+This prints ``{'a': "'str'"}``. This shouldn't really be considered
+a "quirk"; it's mentioned here simply because it might be surprising.
diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst
index f4d08e75d946422..ae5bab90bf81315 100644
--- a/Doc/howto/argparse.rst
+++ b/Doc/howto/argparse.rst
@@ -1,10 +1,12 @@
+.. _argparse-tutorial:
+
*****************
Argparse Tutorial
*****************
:author: Tshepang Mbambo
-.. _argparse-tutorial:
+.. currentmodule:: argparse
This tutorial is intended to be a gentle introduction to :mod:`argparse`, the
recommended command-line parsing module in the Python standard library.
@@ -12,7 +14,7 @@ recommended command-line parsing module in the Python standard library.
.. note::
There are two other modules that fulfill the same task, namely
- :mod:`getopt` (an equivalent for :c:func:`getopt` from the C
+ :mod:`getopt` (an equivalent for ``getopt()`` from the C
language) and the deprecated :mod:`optparse`.
Note also that :mod:`argparse` is based on :mod:`optparse`,
and therefore very similar in terms of usage.
@@ -79,16 +81,16 @@ Following is a result of running the code:
.. code-block:: shell-session
- $ python3 prog.py
- $ python3 prog.py --help
+ $ python prog.py
+ $ python prog.py --help
usage: prog.py [-h]
options:
-h, --help show this help message and exit
- $ python3 prog.py --verbose
+ $ python prog.py --verbose
usage: prog.py [-h]
prog.py: error: unrecognized arguments: --verbose
- $ python3 prog.py foo
+ $ python prog.py foo
usage: prog.py [-h]
prog.py: error: unrecognized arguments: foo
@@ -121,10 +123,10 @@ And running the code:
.. code-block:: shell-session
- $ python3 prog.py
+ $ python prog.py
usage: prog.py [-h] echo
prog.py: error: the following arguments are required: echo
- $ python3 prog.py --help
+ $ python prog.py --help
usage: prog.py [-h] echo
positional arguments:
@@ -132,18 +134,18 @@ And running the code:
options:
-h, --help show this help message and exit
- $ python3 prog.py foo
+ $ python prog.py foo
foo
Here is what's happening:
-* We've added the :meth:`add_argument` method, which is what we use to specify
+* We've added the :meth:`~ArgumentParser.add_argument` method, which is what we use to specify
which command-line options the program is willing to accept. In this case,
I've named it ``echo`` so that it's in line with its function.
* Calling our program now requires us to specify an option.
-* The :meth:`parse_args` method actually returns some data from the
+* The :meth:`~ArgumentParser.parse_args` method actually returns some data from the
options specified, in this case, ``echo``.
* The variable is some form of 'magic' that :mod:`argparse` performs for free
@@ -166,7 +168,7 @@ And we get:
.. code-block:: shell-session
- $ python3 prog.py -h
+ $ python prog.py -h
usage: prog.py [-h] echo
positional arguments:
@@ -187,7 +189,7 @@ Following is a result of running the code:
.. code-block:: shell-session
- $ python3 prog.py 4
+ $ python prog.py 4
Traceback (most recent call last):
File "prog.py", line 5, in
print(args.square**2)
@@ -208,9 +210,9 @@ Following is a result of running the code:
.. code-block:: shell-session
- $ python3 prog.py 4
+ $ python prog.py 4
16
- $ python3 prog.py four
+ $ python prog.py four
usage: prog.py [-h] square
prog.py: error: argument square: invalid int value: 'four'
@@ -235,17 +237,17 @@ And the output:
.. code-block:: shell-session
- $ python3 prog.py --verbosity 1
+ $ python prog.py --verbosity 1
verbosity turned on
- $ python3 prog.py
- $ python3 prog.py --help
+ $ python prog.py
+ $ python prog.py --help
usage: prog.py [-h] [--verbosity VERBOSITY]
options:
-h, --help show this help message and exit
--verbosity VERBOSITY
increase output verbosity
- $ python3 prog.py --verbosity
+ $ python prog.py --verbosity
usage: prog.py [-h] [--verbosity VERBOSITY]
prog.py: error: argument --verbosity: expected one argument
@@ -256,7 +258,7 @@ Here is what is happening:
* To show that the option is actually optional, there is no error when running
the program without it. Note that by default, if an optional argument isn't
- used, the relevant variable, in this case :attr:`args.verbosity`, is
+ used, the relevant variable, in this case ``args.verbosity``, is
given ``None`` as a value, which is the reason it fails the truth
test of the :keyword:`if` statement.
@@ -281,12 +283,12 @@ And the output:
.. code-block:: shell-session
- $ python3 prog.py --verbose
+ $ python prog.py --verbose
verbosity turned on
- $ python3 prog.py --verbose 1
+ $ python prog.py --verbose 1
usage: prog.py [-h] [--verbose]
prog.py: error: unrecognized arguments: 1
- $ python3 prog.py --help
+ $ python prog.py --help
usage: prog.py [-h] [--verbose]
options:
@@ -299,7 +301,7 @@ Here is what is happening:
We even changed the name of the option to match that idea.
Note that we now specify a new keyword, ``action``, and give it the value
``"store_true"``. This means that, if the option is specified,
- assign the value ``True`` to :data:`args.verbose`.
+ assign the value ``True`` to ``args.verbose``.
Not specifying it implies ``False``.
* It complains when you specify a value, in true spirit of what flags
@@ -327,9 +329,9 @@ And here goes:
.. code-block:: shell-session
- $ python3 prog.py -v
+ $ python prog.py -v
verbosity turned on
- $ python3 prog.py --help
+ $ python prog.py --help
usage: prog.py [-h] [-v]
options:
@@ -361,14 +363,14 @@ And now the output:
.. code-block:: shell-session
- $ python3 prog.py
+ $ python prog.py
usage: prog.py [-h] [-v] square
prog.py: error: the following arguments are required: square
- $ python3 prog.py 4
+ $ python prog.py 4
16
- $ python3 prog.py 4 --verbose
+ $ python prog.py 4 --verbose
the square of 4 equals 16
- $ python3 prog.py --verbose 4
+ $ python prog.py --verbose 4
the square of 4 equals 16
* We've brought back a positional argument, hence the complaint.
@@ -397,16 +399,16 @@ And the output:
.. code-block:: shell-session
- $ python3 prog.py 4
+ $ python prog.py 4
16
- $ python3 prog.py 4 -v
+ $ python prog.py 4 -v
usage: prog.py [-h] [-v VERBOSITY] square
prog.py: error: argument -v/--verbosity: expected one argument
- $ python3 prog.py 4 -v 1
+ $ python prog.py 4 -v 1
4^2 == 16
- $ python3 prog.py 4 -v 2
+ $ python prog.py 4 -v 2
the square of 4 equals 16
- $ python3 prog.py 4 -v 3
+ $ python prog.py 4 -v 3
16
These all look good except the last one, which exposes a bug in our program.
@@ -431,10 +433,10 @@ And the output:
.. code-block:: shell-session
- $ python3 prog.py 4 -v 3
+ $ python prog.py 4 -v 3
usage: prog.py [-h] [-v {0,1,2}] square
prog.py: error: argument -v/--verbosity: invalid choice: 3 (choose from 0, 1, 2)
- $ python3 prog.py 4 -h
+ $ python prog.py 4 -h
usage: prog.py [-h] [-v {0,1,2}] square
positional arguments:
@@ -473,18 +475,18 @@ to count the number of occurrences of specific options.
.. code-block:: shell-session
- $ python3 prog.py 4
+ $ python prog.py 4
16
- $ python3 prog.py 4 -v
+ $ python prog.py 4 -v
4^2 == 16
- $ python3 prog.py 4 -vv
+ $ python prog.py 4 -vv
the square of 4 equals 16
- $ python3 prog.py 4 --verbosity --verbosity
+ $ python prog.py 4 --verbosity --verbosity
the square of 4 equals 16
- $ python3 prog.py 4 -v 1
+ $ python prog.py 4 -v 1
usage: prog.py [-h] [-v] square
prog.py: error: unrecognized arguments: 1
- $ python3 prog.py 4 -h
+ $ python prog.py 4 -h
usage: prog.py [-h] [-v] square
positional arguments:
@@ -493,7 +495,7 @@ to count the number of occurrences of specific options.
options:
-h, --help show this help message and exit
-v, --verbosity increase output verbosity
- $ python3 prog.py 4 -vvv
+ $ python prog.py 4 -vvv
16
* Yes, it's now more of a flag (similar to ``action="store_true"``) in the
@@ -540,11 +542,11 @@ And this is what it gives:
.. code-block:: shell-session
- $ python3 prog.py 4 -vvv
+ $ python prog.py 4 -vvv
the square of 4 equals 16
- $ python3 prog.py 4 -vvvv
+ $ python prog.py 4 -vvvv
the square of 4 equals 16
- $ python3 prog.py 4
+ $ python prog.py 4
Traceback (most recent call last):
File "prog.py", line 11, in
if args.verbosity >= 2:
@@ -584,7 +586,7 @@ And:
.. code-block:: shell-session
- $ python3 prog.py 4
+ $ python prog.py 4
16
You can go quite far just with what we've learned so far,
@@ -617,10 +619,10 @@ Output:
.. code-block:: shell-session
- $ python3 prog.py
+ $ python prog.py
usage: prog.py [-h] [-v] x y
prog.py: error: the following arguments are required: x, y
- $ python3 prog.py -h
+ $ python prog.py -h
usage: prog.py [-h] [-v] x y
positional arguments:
@@ -630,7 +632,7 @@ Output:
options:
-h, --help show this help message and exit
-v, --verbosity
- $ python3 prog.py 4 2 -v
+ $ python prog.py 4 2 -v
4^2 == 16
@@ -655,11 +657,11 @@ Output:
.. code-block:: shell-session
- $ python3 prog.py 4 2
+ $ python prog.py 4 2
16
- $ python3 prog.py 4 2 -v
+ $ python prog.py 4 2 -v
4^2 == 16
- $ python3 prog.py 4 2 -vv
+ $ python prog.py 4 2 -vv
Running 'prog.py'
4^2 == 16
@@ -698,7 +700,7 @@ Conflicting options
So far, we have been working with two methods of an
:class:`argparse.ArgumentParser` instance. Let's introduce a third one,
-:meth:`add_mutually_exclusive_group`. It allows for us to specify options that
+:meth:`~ArgumentParser.add_mutually_exclusive_group`. It allows for us to specify options that
conflict with each other. Let's also change the rest of the program so that
the new functionality makes more sense:
we'll introduce the ``--quiet`` option,
@@ -727,16 +729,16 @@ demonstration. Anyways, here's the output:
.. code-block:: shell-session
- $ python3 prog.py 4 2
+ $ python prog.py 4 2
4^2 == 16
- $ python3 prog.py 4 2 -q
+ $ python prog.py 4 2 -q
16
- $ python3 prog.py 4 2 -v
+ $ python prog.py 4 2 -v
4 to the power 2 equals 16
- $ python3 prog.py 4 2 -vq
+ $ python prog.py 4 2 -vq
usage: prog.py [-h] [-v | -q] x y
prog.py: error: argument -q/--quiet: not allowed with argument -v/--verbose
- $ python3 prog.py 4 2 -v --quiet
+ $ python prog.py 4 2 -v --quiet
usage: prog.py [-h] [-v | -q] x y
prog.py: error: argument -q/--quiet: not allowed with argument -v/--verbose
@@ -771,7 +773,34 @@ but not both at the same time:
.. code-block:: shell-session
- $ python3 prog.py --help
+ $ python prog.py --help
+ usage: prog.py [-h] [-v | -q] x y
+
+ calculate X to the power of Y
+
+ positional arguments:
+ x the base
+ y the exponent
+
+ options:
+ -h, --help show this help message and exit
+ -v, --verbose
+ -q, --quiet
+
+
+How to translate the argparse output
+====================================
+
+The output of the :mod:`argparse` module such as its help text and error
+messages are all made translatable using the :mod:`gettext` module. This
+allows applications to easily localize messages produced by
+:mod:`argparse`. See also :ref:`i18n-howto`.
+
+For instance, in this :mod:`argparse` output:
+
+.. code-block:: shell-session
+
+ $ python prog.py --help
usage: prog.py [-h] [-v | -q] x y
calculate X to the power of Y
@@ -785,6 +814,32 @@ but not both at the same time:
-v, --verbose
-q, --quiet
+The strings ``usage:``, ``positional arguments:``, ``options:`` and
+``show this help message and exit`` are all translatable.
+
+In order to translate these strings, they must first be extracted
+into a ``.po`` file. For example, using `Babel `__,
+run this command:
+
+.. code-block:: shell-session
+
+ $ pybabel extract -o messages.po /usr/lib/python3.12/argparse.py
+
+This command will extract all translatable strings from the :mod:`argparse`
+module and output them into a file named ``messages.po``. This command assumes
+that your Python installation is in ``/usr/lib``.
+
+You can find out the location of the :mod:`argparse` module on your system
+using this script::
+
+ import argparse
+ print(argparse.__file__)
+
+Once the messages in the ``.po`` file are translated and the translations are
+installed using :mod:`gettext`, :mod:`argparse` will be able to display the
+translated messages.
+
+To translate your own strings in the :mod:`argparse` output, use :mod:`gettext`.
Conclusion
==========
diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst
index a97f1d23f53f318..060977246149cfa 100644
--- a/Doc/howto/clinic.rst
+++ b/Doc/howto/clinic.rst
@@ -1,1804 +1,14 @@
-.. highlight:: c
+:orphan:
-.. _howto-clinic:
+.. This page is retained solely for existing links to /howto/clinic.html.
+ Direct readers to the devguide.
**********************
Argument Clinic How-To
**********************
-:author: Larry Hastings
+.. note::
-.. topic:: Abstract
-
- Argument Clinic is a preprocessor for CPython C files.
- Its purpose is to automate all the boilerplate involved
- with writing argument parsing code for "builtins".
- This document shows you how to convert your first C
- function to work with Argument Clinic, and then introduces
- some advanced topics on Argument Clinic usage.
-
- Currently Argument Clinic is considered internal-only
- for CPython. Its use is not supported for files outside
- CPython, and no guarantees are made regarding backwards
- compatibility for future versions. In other words: if you
- maintain an external C extension for CPython, you're welcome
- to experiment with Argument Clinic in your own code. But the
- version of Argument Clinic that ships with the next version
- of CPython *could* be totally incompatible and break all your code.
-
-The Goals Of Argument Clinic
-============================
-
-Argument Clinic's primary goal
-is to take over responsibility for all argument parsing code
-inside CPython. This means that, when you convert a function
-to work with Argument Clinic, that function should no longer
-do any of its own argument parsing—the code generated by
-Argument Clinic should be a "black box" to you, where CPython
-calls in at the top, and your code gets called at the bottom,
-with ``PyObject *args`` (and maybe ``PyObject *kwargs``)
-magically converted into the C variables and types you need.
-
-In order for Argument Clinic to accomplish its primary goal,
-it must be easy to use. Currently, working with CPython's
-argument parsing library is a chore, requiring maintaining
-redundant information in a surprising number of places.
-When you use Argument Clinic, you don't have to repeat yourself.
-
-Obviously, no one would want to use Argument Clinic unless
-it's solving their problem—and without creating new problems of
-its own.
-So it's paramount that Argument Clinic generate correct code.
-It'd be nice if the code was faster, too, but at the very least
-it should not introduce a major speed regression. (Eventually Argument
-Clinic *should* make a major speedup possible—we could
-rewrite its code generator to produce tailor-made argument
-parsing code, rather than calling the general-purpose CPython
-argument parsing library. That would make for the fastest
-argument parsing possible!)
-
-Additionally, Argument Clinic must be flexible enough to
-work with any approach to argument parsing. Python has
-some functions with some very strange parsing behaviors;
-Argument Clinic's goal is to support all of them.
-
-Finally, the original motivation for Argument Clinic was
-to provide introspection "signatures" for CPython builtins.
-It used to be, the introspection query functions would throw
-an exception if you passed in a builtin. With Argument
-Clinic, that's a thing of the past!
-
-One idea you should keep in mind, as you work with
-Argument Clinic: the more information you give it, the
-better job it'll be able to do.
-Argument Clinic is admittedly relatively simple right
-now. But as it evolves it will get more sophisticated,
-and it should be able to do many interesting and smart
-things with all the information you give it.
-
-
-Basic Concepts And Usage
-========================
-
-Argument Clinic ships with CPython; you'll find it in ``Tools/clinic/clinic.py``.
-If you run that script, specifying a C file as an argument:
-
-.. code-block:: shell-session
-
- $ python3 Tools/clinic/clinic.py foo.c
-
-Argument Clinic will scan over the file looking for lines that
-look exactly like this:
-
-.. code-block:: none
-
- /*[clinic input]
-
-When it finds one, it reads everything up to a line that looks
-exactly like this:
-
-.. code-block:: none
-
- [clinic start generated code]*/
-
-Everything in between these two lines is input for Argument Clinic.
-All of these lines, including the beginning and ending comment
-lines, are collectively called an Argument Clinic "block".
-
-When Argument Clinic parses one of these blocks, it
-generates output. This output is rewritten into the C file
-immediately after the block, followed by a comment containing a checksum.
-The Argument Clinic block now looks like this:
-
-.. code-block:: none
-
- /*[clinic input]
- ... clinic input goes here ...
- [clinic start generated code]*/
- ... clinic output goes here ...
- /*[clinic end generated code: checksum=...]*/
-
-If you run Argument Clinic on the same file a second time, Argument Clinic
-will discard the old output and write out the new output with a fresh checksum
-line. However, if the input hasn't changed, the output won't change either.
-
-You should never modify the output portion of an Argument Clinic block. Instead,
-change the input until it produces the output you want. (That's the purpose of the
-checksum—to detect if someone changed the output, as these edits would be lost
-the next time Argument Clinic writes out fresh output.)
-
-For the sake of clarity, here's the terminology we'll use with Argument Clinic:
-
-* The first line of the comment (``/*[clinic input]``) is the *start line*.
-* The last line of the initial comment (``[clinic start generated code]*/``) is the *end line*.
-* The last line (``/*[clinic end generated code: checksum=...]*/``) is the *checksum line*.
-* In between the start line and the end line is the *input*.
-* In between the end line and the checksum line is the *output*.
-* All the text collectively, from the start line to the checksum line inclusively,
- is the *block*. (A block that hasn't been successfully processed by Argument
- Clinic yet doesn't have output or a checksum line, but it's still considered
- a block.)
-
-
-Converting Your First Function
-==============================
-
-The best way to get a sense of how Argument Clinic works is to
-convert a function to work with it. Here, then, are the bare
-minimum steps you'd need to follow to convert a function to
-work with Argument Clinic. Note that for code you plan to
-check in to CPython, you really should take the conversion farther,
-using some of the advanced concepts you'll see later on in
-the document (like "return converters" and "self converters").
-But we'll keep it simple for this walkthrough so you can learn.
-
-Let's dive in!
-
-0. Make sure you're working with a freshly updated checkout
- of the CPython trunk.
-
-1. Find a Python builtin that calls either :c:func:`PyArg_ParseTuple`
- or :c:func:`PyArg_ParseTupleAndKeywords`, and hasn't been converted
- to work with Argument Clinic yet.
- For my example I'm using ``_pickle.Pickler.dump()``.
-
-2. If the call to the ``PyArg_Parse`` function uses any of the
- following format units:
-
- .. code-block:: none
-
- O&
- O!
- es
- es#
- et
- et#
-
- or if it has multiple calls to :c:func:`PyArg_ParseTuple`,
- you should choose a different function. Argument Clinic *does*
- support all of these scenarios. But these are advanced
- topics—let's do something simpler for your first function.
-
- Also, if the function has multiple calls to :c:func:`PyArg_ParseTuple`
- or :c:func:`PyArg_ParseTupleAndKeywords` where it supports different
- types for the same argument, or if the function uses something besides
- PyArg_Parse functions to parse its arguments, it probably
- isn't suitable for conversion to Argument Clinic. Argument Clinic
- doesn't support generic functions or polymorphic parameters.
-
-3. Add the following boilerplate above the function, creating our block::
-
- /*[clinic input]
- [clinic start generated code]*/
-
-4. Cut the docstring and paste it in between the ``[clinic]`` lines,
- removing all the junk that makes it a properly quoted C string.
- When you're done you should have just the text, based at the left
- margin, with no line wider than 80 characters.
- (Argument Clinic will preserve indents inside the docstring.)
-
- If the old docstring had a first line that looked like a function
- signature, throw that line away. (The docstring doesn't need it
- anymore—when you use ``help()`` on your builtin in the future,
- the first line will be built automatically based on the function's
- signature.)
-
- Sample::
-
- /*[clinic input]
- Write a pickled representation of obj to the open file.
- [clinic start generated code]*/
-
-5. If your docstring doesn't have a "summary" line, Argument Clinic will
- complain. So let's make sure it has one. The "summary" line should
- be a paragraph consisting of a single 80-column line
- at the beginning of the docstring.
-
- (Our example docstring consists solely of a summary line, so the sample
- code doesn't have to change for this step.)
-
-6. Above the docstring, enter the name of the function, followed
- by a blank line. This should be the Python name of the function,
- and should be the full dotted path
- to the function—it should start with the name of the module,
- include any sub-modules, and if the function is a method on
- a class it should include the class name too.
-
- Sample::
-
- /*[clinic input]
- _pickle.Pickler.dump
-
- Write a pickled representation of obj to the open file.
- [clinic start generated code]*/
-
-7. If this is the first time that module or class has been used with Argument
- Clinic in this C file,
- you must declare the module and/or class. Proper Argument Clinic hygiene
- prefers declaring these in a separate block somewhere near the
- top of the C file, in the same way that include files and statics go at
- the top. (In our sample code we'll just show the two blocks next to
- each other.)
-
- The name of the class and module should be the same as the one
- seen by Python. Check the name defined in the :c:type:`PyModuleDef`
- or :c:type:`PyTypeObject` as appropriate.
-
- When you declare a class, you must also specify two aspects of its type
- in C: the type declaration you'd use for a pointer to an instance of
- this class, and a pointer to the :c:type:`PyTypeObject` for this class.
-
- Sample::
-
- /*[clinic input]
- module _pickle
- class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
- [clinic start generated code]*/
-
- /*[clinic input]
- _pickle.Pickler.dump
-
- Write a pickled representation of obj to the open file.
- [clinic start generated code]*/
-
-
-
-
-8. Declare each of the parameters to the function. Each parameter
- should get its own line. All the parameter lines should be
- indented from the function name and the docstring.
-
- The general form of these parameter lines is as follows:
-
- .. code-block:: none
-
- name_of_parameter: converter
-
- If the parameter has a default value, add that after the
- converter:
-
- .. code-block:: none
-
- name_of_parameter: converter = default_value
-
- Argument Clinic's support for "default values" is quite sophisticated;
- please see :ref:`the section below on default values `
- for more information.
-
- Add a blank line below the parameters.
-
- What's a "converter"? It establishes both the type
- of the variable used in C, and the method to convert the Python
- value into a C value at runtime.
- For now you're going to use what's called a "legacy converter"—a
- convenience syntax intended to make porting old code into Argument
- Clinic easier.
-
- For each parameter, copy the "format unit" for that
- parameter from the ``PyArg_Parse()`` format argument and
- specify *that* as its converter, as a quoted
- string. ("format unit" is the formal name for the one-to-three
- character substring of the ``format`` parameter that tells
- the argument parsing function what the type of the variable
- is and how to convert it. For more on format units please
- see :ref:`arg-parsing`.)
-
- For multicharacter format units like ``z#``, use the
- entire two-or-three character string.
-
- Sample::
-
- /*[clinic input]
- module _pickle
- class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
- [clinic start generated code]*/
-
- /*[clinic input]
- _pickle.Pickler.dump
-
- obj: 'O'
-
- Write a pickled representation of obj to the open file.
- [clinic start generated code]*/
-
-9. If your function has ``|`` in the format string, meaning some
- parameters have default values, you can ignore it. Argument
- Clinic infers which parameters are optional based on whether
- or not they have default values.
-
- If your function has ``$`` in the format string, meaning it
- takes keyword-only arguments, specify ``*`` on a line by
- itself before the first keyword-only argument, indented the
- same as the parameter lines.
-
- (``_pickle.Pickler.dump`` has neither, so our sample is unchanged.)
-
-
-10. If the existing C function calls :c:func:`PyArg_ParseTuple`
- (as opposed to :c:func:`PyArg_ParseTupleAndKeywords`), then all its
- arguments are positional-only.
-
- To mark all parameters as positional-only in Argument Clinic,
- add a ``/`` on a line by itself after the last parameter,
- indented the same as the parameter lines.
-
- Currently this is all-or-nothing; either all parameters are
- positional-only, or none of them are. (In the future Argument
- Clinic may relax this restriction.)
-
- Sample::
-
- /*[clinic input]
- module _pickle
- class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
- [clinic start generated code]*/
-
- /*[clinic input]
- _pickle.Pickler.dump
-
- obj: 'O'
- /
-
- Write a pickled representation of obj to the open file.
- [clinic start generated code]*/
-
-11. It's helpful to write a per-parameter docstring for each parameter.
- But per-parameter docstrings are optional; you can skip this step
- if you prefer.
-
- Here's how to add a per-parameter docstring. The first line
- of the per-parameter docstring must be indented further than the
- parameter definition. The left margin of this first line establishes
- the left margin for the whole per-parameter docstring; all the text
- you write will be outdented by this amount. You can write as much
- text as you like, across multiple lines if you wish.
-
- Sample::
-
- /*[clinic input]
- module _pickle
- class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
- [clinic start generated code]*/
-
- /*[clinic input]
- _pickle.Pickler.dump
-
- obj: 'O'
- The object to be pickled.
- /
-
- Write a pickled representation of obj to the open file.
- [clinic start generated code]*/
-
-12. Save and close the file, then run ``Tools/clinic/clinic.py`` on
- it. With luck everything worked---your block now has output, and
- a ``.c.h`` file has been generated! Reopen the file in your
- text editor to see::
-
- /*[clinic input]
- _pickle.Pickler.dump
-
- obj: 'O'
- The object to be pickled.
- /
-
- Write a pickled representation of obj to the open file.
- [clinic start generated code]*/
-
- static PyObject *
- _pickle_Pickler_dump(PicklerObject *self, PyObject *obj)
- /*[clinic end generated code: output=87ecad1261e02ac7 input=552eb1c0f52260d9]*/
-
- Obviously, if Argument Clinic didn't produce any output, it's because
- it found an error in your input. Keep fixing your errors and retrying
- until Argument Clinic processes your file without complaint.
-
- For readability, most of the glue code has been generated to a ``.c.h``
- file. You'll need to include that in your original ``.c`` file,
- typically right after the clinic module block::
-
- #include "clinic/_pickle.c.h"
-
-13. Double-check that the argument-parsing code Argument Clinic generated
- looks basically the same as the existing code.
-
- First, ensure both places use the same argument-parsing function.
- The existing code must call either
- :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_ParseTupleAndKeywords`;
- ensure that the code generated by Argument Clinic calls the
- *exact* same function.
-
- Second, the format string passed in to :c:func:`PyArg_ParseTuple` or
- :c:func:`PyArg_ParseTupleAndKeywords` should be *exactly* the same
- as the hand-written one in the existing function, up to the colon
- or semi-colon.
-
- (Argument Clinic always generates its format strings
- with a ``:`` followed by the name of the function. If the
- existing code's format string ends with ``;``, to provide
- usage help, this change is harmless—don't worry about it.)
-
- Third, for parameters whose format units require two arguments
- (like a length variable, or an encoding string, or a pointer
- to a conversion function), ensure that the second argument is
- *exactly* the same between the two invocations.
-
- Fourth, inside the output portion of the block you'll find a preprocessor
- macro defining the appropriate static :c:type:`PyMethodDef` structure for
- this builtin::
-
- #define __PICKLE_PICKLER_DUMP_METHODDEF \
- {"dump", (PyCFunction)__pickle_Pickler_dump, METH_O, __pickle_Pickler_dump__doc__},
-
- This static structure should be *exactly* the same as the existing static
- :c:type:`PyMethodDef` structure for this builtin.
-
- If any of these items differ in *any way*,
- adjust your Argument Clinic function specification and rerun
- ``Tools/clinic/clinic.py`` until they *are* the same.
-
-
-14. Notice that the last line of its output is the declaration
- of your "impl" function. This is where the builtin's implementation goes.
- Delete the existing prototype of the function you're modifying, but leave
- the opening curly brace. Now delete its argument parsing code and the
- declarations of all the variables it dumps the arguments into.
- Notice how the Python arguments are now arguments to this impl function;
- if the implementation used different names for these variables, fix it.
-
- Let's reiterate, just because it's kind of weird. Your code should now
- look like this::
-
- static return_type
- your_function_impl(...)
- /*[clinic end generated code: checksum=...]*/
- {
- ...
-
- Argument Clinic generated the checksum line and the function prototype just
- above it. You should write the opening (and closing) curly braces for the
- function, and the implementation inside.
-
- Sample::
-
- /*[clinic input]
- module _pickle
- class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
- [clinic start generated code]*/
- /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
-
- /*[clinic input]
- _pickle.Pickler.dump
-
- obj: 'O'
- The object to be pickled.
- /
-
- Write a pickled representation of obj to the open file.
- [clinic start generated code]*/
-
- PyDoc_STRVAR(__pickle_Pickler_dump__doc__,
- "Write a pickled representation of obj to the open file.\n"
- "\n"
- ...
- static PyObject *
- _pickle_Pickler_dump_impl(PicklerObject *self, PyObject *obj)
- /*[clinic end generated code: checksum=3bd30745bf206a48f8b576a1da3d90f55a0a4187]*/
- {
- /* Check whether the Pickler was initialized correctly (issue3664).
- Developers often forget to call __init__() in their subclasses, which
- would trigger a segfault without this check. */
- if (self->write == NULL) {
- PyErr_Format(PicklingError,
- "Pickler.__init__() was not called by %s.__init__()",
- Py_TYPE(self)->tp_name);
- return NULL;
- }
-
- if (_Pickler_ClearBuffer(self) < 0)
- return NULL;
-
- ...
-
-15. Remember the macro with the :c:type:`PyMethodDef` structure for this
- function? Find the existing :c:type:`PyMethodDef` structure for this
- function and replace it with a reference to the macro. (If the builtin
- is at module scope, this will probably be very near the end of the file;
- if the builtin is a class method, this will probably be below but relatively
- near to the implementation.)
-
- Note that the body of the macro contains a trailing comma. So when you
- replace the existing static :c:type:`PyMethodDef` structure with the macro,
- *don't* add a comma to the end.
-
- Sample::
-
- static struct PyMethodDef Pickler_methods[] = {
- __PICKLE_PICKLER_DUMP_METHODDEF
- __PICKLE_PICKLER_CLEAR_MEMO_METHODDEF
- {NULL, NULL} /* sentinel */
- };
-
-
-16. Argument Clinic may generate new instances of ``_Py_ID``. For example::
-
- &_Py_ID(new_unique_py_id)
-
- If it does, you'll have to run ``Tools/scripts/generate_global_objects.py``
- to regenerate the list of precompiled identifiers at this point.
-
-
-17. Compile, then run the relevant portions of the regression-test suite.
- This change should not introduce any new compile-time warnings or errors,
- and there should be no externally visible change to Python's behavior.
-
- Well, except for one difference: ``inspect.signature()`` run on your function
- should now provide a valid signature!
-
- Congratulations, you've ported your first function to work with Argument Clinic!
-
-Advanced Topics
-===============
-
-Now that you've had some experience working with Argument Clinic, it's time
-for some advanced topics.
-
-
-Symbolic default values
------------------------
-
-The default value you provide for a parameter can't be any arbitrary
-expression. Currently the following are explicitly supported:
-
-* Numeric constants (integer and float)
-* String constants
-* ``True``, ``False``, and ``None``
-* Simple symbolic constants like ``sys.maxsize``, which must
- start with the name of the module
-
-(In the future, this may need to get even more elaborate,
-to allow full expressions like ``CONSTANT - 1``.)
-
-
-Renaming the C functions and variables generated by Argument Clinic
--------------------------------------------------------------------
-
-Argument Clinic automatically names the functions it generates for you.
-Occasionally this may cause a problem, if the generated name collides with
-the name of an existing C function. There's an easy solution: override the names
-used for the C functions. Just add the keyword ``"as"``
-to your function declaration line, followed by the function name you wish to use.
-Argument Clinic will use that function name for the base (generated) function,
-then add ``"_impl"`` to the end and use that for the name of the impl function.
-
-For example, if we wanted to rename the C function names generated for
-``pickle.Pickler.dump``, it'd look like this::
-
- /*[clinic input]
- pickle.Pickler.dump as pickler_dumper
-
- ...
-
-The base function would now be named ``pickler_dumper()``,
-and the impl function would now be named ``pickler_dumper_impl()``.
-
-
-Similarly, you may have a problem where you want to give a parameter
-a specific Python name, but that name may be inconvenient in C. Argument
-Clinic allows you to give a parameter different names in Python and in C,
-using the same ``"as"`` syntax::
-
- /*[clinic input]
- pickle.Pickler.dump
-
- obj: object
- file as file_obj: object
- protocol: object = NULL
- *
- fix_imports: bool = True
-
-Here, the name used in Python (in the signature and the ``keywords``
-array) would be ``file``, but the C variable would be named ``file_obj``.
-
-You can use this to rename the ``self`` parameter too!
-
-
-Converting functions using PyArg_UnpackTuple
---------------------------------------------
-
-To convert a function parsing its arguments with :c:func:`PyArg_UnpackTuple`,
-simply write out all the arguments, specifying each as an ``object``. You
-may specify the ``type`` argument to cast the type as appropriate. All
-arguments should be marked positional-only (add a ``/`` on a line by itself
-after the last argument).
-
-Currently the generated code will use :c:func:`PyArg_ParseTuple`, but this
-will change soon.
-
-Optional Groups
----------------
-
-Some legacy functions have a tricky approach to parsing their arguments:
-they count the number of positional arguments, then use a ``switch`` statement
-to call one of several different :c:func:`PyArg_ParseTuple` calls depending on
-how many positional arguments there are. (These functions cannot accept
-keyword-only arguments.) This approach was used to simulate optional
-arguments back before :c:func:`PyArg_ParseTupleAndKeywords` was created.
-
-While functions using this approach can often be converted to
-use :c:func:`PyArg_ParseTupleAndKeywords`, optional arguments, and default values,
-it's not always possible. Some of these legacy functions have
-behaviors :c:func:`PyArg_ParseTupleAndKeywords` doesn't directly support.
-The most obvious example is the builtin function ``range()``, which has
-an optional argument on the *left* side of its required argument!
-Another example is ``curses.window.addch()``, which has a group of two
-arguments that must always be specified together. (The arguments are
-called ``x`` and ``y``; if you call the function passing in ``x``,
-you must also pass in ``y``—and if you don't pass in ``x`` you may not
-pass in ``y`` either.)
-
-In any case, the goal of Argument Clinic is to support argument parsing
-for all existing CPython builtins without changing their semantics.
-Therefore Argument Clinic supports
-this alternate approach to parsing, using what are called *optional groups*.
-Optional groups are groups of arguments that must all be passed in together.
-They can be to the left or the right of the required arguments. They
-can *only* be used with positional-only parameters.
-
-.. note:: Optional groups are *only* intended for use when converting
- functions that make multiple calls to :c:func:`PyArg_ParseTuple`!
- Functions that use *any* other approach for parsing arguments
- should *almost never* be converted to Argument Clinic using
- optional groups. Functions using optional groups currently
- cannot have accurate signatures in Python, because Python just
- doesn't understand the concept. Please avoid using optional
- groups wherever possible.
-
-To specify an optional group, add a ``[`` on a line by itself before
-the parameters you wish to group together, and a ``]`` on a line by itself
-after these parameters. As an example, here's how ``curses.window.addch``
-uses optional groups to make the first two parameters and the last
-parameter optional::
-
- /*[clinic input]
-
- curses.window.addch
-
- [
- x: int
- X-coordinate.
- y: int
- Y-coordinate.
- ]
-
- ch: object
- Character to add.
-
- [
- attr: long
- Attributes for the character.
- ]
- /
-
- ...
-
-
-Notes:
-
-* For every optional group, one additional parameter will be passed into the
- impl function representing the group. The parameter will be an int named
- ``group_{direction}_{number}``,
- where ``{direction}`` is either ``right`` or ``left`` depending on whether the group
- is before or after the required parameters, and ``{number}`` is a monotonically
- increasing number (starting at 1) indicating how far away the group is from
- the required parameters. When the impl is called, this parameter will be set
- to zero if this group was unused, and set to non-zero if this group was used.
- (By used or unused, I mean whether or not the parameters received arguments
- in this invocation.)
-
-* If there are no required arguments, the optional groups will behave
- as if they're to the right of the required arguments.
-
-* In the case of ambiguity, the argument parsing code
- favors parameters on the left (before the required parameters).
-
-* Optional groups can only contain positional-only parameters.
-
-* Optional groups are *only* intended for legacy code. Please do not
- use optional groups for new code.
-
-
-Using real Argument Clinic converters, instead of "legacy converters"
----------------------------------------------------------------------
-
-To save time, and to minimize how much you need to learn
-to achieve your first port to Argument Clinic, the walkthrough above tells
-you to use "legacy converters". "Legacy converters" are a convenience,
-designed explicitly to make porting existing code to Argument Clinic
-easier. And to be clear, their use is acceptable when porting code for
-Python 3.4.
-
-However, in the long term we probably want all our blocks to
-use Argument Clinic's real syntax for converters. Why? A couple
-reasons:
-
-* The proper converters are far easier to read and clearer in their intent.
-* There are some format units that are unsupported as "legacy converters",
- because they require arguments, and the legacy converter syntax doesn't
- support specifying arguments.
-* In the future we may have a new argument parsing library that isn't
- restricted to what :c:func:`PyArg_ParseTuple` supports; this flexibility
- won't be available to parameters using legacy converters.
-
-Therefore, if you don't mind a little extra effort, please use the normal
-converters instead of legacy converters.
-
-In a nutshell, the syntax for Argument Clinic (non-legacy) converters
-looks like a Python function call. However, if there are no explicit
-arguments to the function (all functions take their default values),
-you may omit the parentheses. Thus ``bool`` and ``bool()`` are exactly
-the same converters.
-
-All arguments to Argument Clinic converters are keyword-only.
-All Argument Clinic converters accept the following arguments:
-
- ``c_default``
- The default value for this parameter when defined in C.
- Specifically, this will be the initializer for the variable declared
- in the "parse function". See :ref:`the section on default values `
- for how to use this.
- Specified as a string.
-
- ``annotation``
- The annotation value for this parameter. Not currently supported,
- because :pep:`8` mandates that the Python library may not use
- annotations.
-
-In addition, some converters accept additional arguments. Here is a list
-of these arguments, along with their meanings:
-
- ``accept``
- A set of Python types (and possibly pseudo-types);
- this restricts the allowable Python argument to values of these types.
- (This is not a general-purpose facility; as a rule it only supports
- specific lists of types as shown in the legacy converter table.)
-
- To accept ``None``, add ``NoneType`` to this set.
-
- ``bitwise``
- Only supported for unsigned integers. The native integer value of this
- Python argument will be written to the parameter without any range checking,
- even for negative values.
-
- ``converter``
- Only supported by the ``object`` converter. Specifies the name of a
- :ref:`C "converter function" `
- to use to convert this object to a native type.
-
- ``encoding``
- Only supported for strings. Specifies the encoding to use when converting
- this string from a Python str (Unicode) value into a C ``char *`` value.
-
-
- ``subclass_of``
- Only supported for the ``object`` converter. Requires that the Python
- value be a subclass of a Python type, as expressed in C.
-
- ``type``
- Only supported for the ``object`` and ``self`` converters. Specifies
- the C type that will be used to declare the variable. Default value is
- ``"PyObject *"``.
-
- ``zeroes``
- Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are
- permitted inside the value. The length of the string will be passed in
- to the impl function, just after the string parameter, as a parameter named
- ``_length``.
-
-Please note, not every possible combination of arguments will work.
-Usually these arguments are implemented by specific ``PyArg_ParseTuple``
-*format units*, with specific behavior. For example, currently you cannot
-call ``unsigned_short`` without also specifying ``bitwise=True``.
-Although it's perfectly reasonable to think this would work, these semantics don't
-map to any existing format unit. So Argument Clinic doesn't support it. (Or, at
-least, not yet.)
-
-Below is a table showing the mapping of legacy converters into real
-Argument Clinic converters. On the left is the legacy converter,
-on the right is the text you'd replace it with.
-
-========= =================================================================================
-``'B'`` ``unsigned_char(bitwise=True)``
-``'b'`` ``unsigned_char``
-``'c'`` ``char``
-``'C'`` ``int(accept={str})``
-``'d'`` ``double``
-``'D'`` ``Py_complex``
-``'es'`` ``str(encoding='name_of_encoding')``
-``'es#'`` ``str(encoding='name_of_encoding', zeroes=True)``
-``'et'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str})``
-``'et#'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str}, zeroes=True)``
-``'f'`` ``float``
-``'h'`` ``short``
-``'H'`` ``unsigned_short(bitwise=True)``
-``'i'`` ``int``
-``'I'`` ``unsigned_int(bitwise=True)``
-``'k'`` ``unsigned_long(bitwise=True)``
-``'K'`` ``unsigned_long_long(bitwise=True)``
-``'l'`` ``long``
-``'L'`` ``long long``
-``'n'`` ``Py_ssize_t``
-``'O'`` ``object``
-``'O!'`` ``object(subclass_of='&PySomething_Type')``
-``'O&'`` ``object(converter='name_of_c_function')``
-``'p'`` ``bool``
-``'S'`` ``PyBytesObject``
-``'s'`` ``str``
-``'s#'`` ``str(zeroes=True)``
-``'s*'`` ``Py_buffer(accept={buffer, str})``
-``'U'`` ``unicode``
-``'u'`` ``wchar_t``
-``'u#'`` ``wchar_t(zeroes=True)``
-``'w*'`` ``Py_buffer(accept={rwbuffer})``
-``'Y'`` ``PyByteArrayObject``
-``'y'`` ``str(accept={bytes})``
-``'y#'`` ``str(accept={robuffer}, zeroes=True)``
-``'y*'`` ``Py_buffer``
-``'Z'`` ``wchar_t(accept={str, NoneType})``
-``'Z#'`` ``wchar_t(accept={str, NoneType}, zeroes=True)``
-``'z'`` ``str(accept={str, NoneType})``
-``'z#'`` ``str(accept={str, NoneType}, zeroes=True)``
-``'z*'`` ``Py_buffer(accept={buffer, str, NoneType})``
-========= =================================================================================
-
-As an example, here's our sample ``pickle.Pickler.dump`` using the proper
-converter::
-
- /*[clinic input]
- pickle.Pickler.dump
-
- obj: object
- The object to be pickled.
- /
-
- Write a pickled representation of obj to the open file.
- [clinic start generated code]*/
-
-One advantage of real converters is that they're more flexible than legacy
-converters. For example, the ``unsigned_int`` converter (and all the
-``unsigned_`` converters) can be specified without ``bitwise=True``. Their
-default behavior performs range checking on the value, and they won't accept
-negative numbers. You just can't do that with a legacy converter!
-
-Argument Clinic will show you all the converters it has
-available. For each converter it'll show you all the parameters
-it accepts, along with the default value for each parameter.
-Just run ``Tools/clinic/clinic.py --converters`` to see the full list.
-
-Py_buffer
----------
-
-When using the ``Py_buffer`` converter
-(or the ``'s*'``, ``'w*'``, ``'*y'``, or ``'z*'`` legacy converters),
-you *must* not call :c:func:`PyBuffer_Release` on the provided buffer.
-Argument Clinic generates code that does it for you (in the parsing function).
-
-
-
-Advanced converters
--------------------
-
-Remember those format units you skipped for your first
-time because they were advanced? Here's how to handle those too.
-
-The trick is, all those format units take arguments—either
-conversion functions, or types, or strings specifying an encoding.
-(But "legacy converters" don't support arguments. That's why we
-skipped them for your first function.) The argument you specified
-to the format unit is now an argument to the converter; this
-argument is either ``converter`` (for ``O&``), ``subclass_of`` (for ``O!``),
-or ``encoding`` (for all the format units that start with ``e``).
-
-When using ``subclass_of``, you may also want to use the other
-custom argument for ``object()``: ``type``, which lets you set the type
-actually used for the parameter. For example, if you want to ensure
-that the object is a subclass of ``PyUnicode_Type``, you probably want
-to use the converter ``object(type='PyUnicodeObject *', subclass_of='&PyUnicode_Type')``.
-
-One possible problem with using Argument Clinic: it takes away some possible
-flexibility for the format units starting with ``e``. When writing a
-``PyArg_Parse`` call by hand, you could theoretically decide at runtime what
-encoding string to pass in to :c:func:`PyArg_ParseTuple`. But now this string must
-be hard-coded at Argument-Clinic-preprocessing-time. This limitation is deliberate;
-it made supporting this format unit much easier, and may allow for future optimizations.
-This restriction doesn't seem unreasonable; CPython itself always passes in static
-hard-coded encoding strings for parameters whose format units start with ``e``.
-
-
-.. _default_values:
-
-Parameter default values
-------------------------
-
-Default values for parameters can be any of a number of values.
-At their simplest, they can be string, int, or float literals:
-
-.. code-block:: none
-
- foo: str = "abc"
- bar: int = 123
- bat: float = 45.6
-
-They can also use any of Python's built-in constants:
-
-.. code-block:: none
-
- yep: bool = True
- nope: bool = False
- nada: object = None
-
-There's also special support for a default value of ``NULL``, and
-for simple expressions, documented in the following sections.
-
-
-The ``NULL`` default value
---------------------------
-
-For string and object parameters, you can set them to ``None`` to indicate
-that there's no default. However, that means the C variable will be
-initialized to ``Py_None``. For convenience's sakes, there's a special
-value called ``NULL`` for just this reason: from Python's perspective it
-behaves like a default value of ``None``, but the C variable is initialized
-with ``NULL``.
-
-Expressions specified as default values
----------------------------------------
-
-The default value for a parameter can be more than just a literal value.
-It can be an entire expression, using math operators and looking up attributes
-on objects. However, this support isn't exactly simple, because of some
-non-obvious semantics.
-
-Consider the following example:
-
-.. code-block:: none
-
- foo: Py_ssize_t = sys.maxsize - 1
-
-``sys.maxsize`` can have different values on different platforms. Therefore
-Argument Clinic can't simply evaluate that expression locally and hard-code it
-in C. So it stores the default in such a way that it will get evaluated at
-runtime, when the user asks for the function's signature.
-
-What namespace is available when the expression is evaluated? It's evaluated
-in the context of the module the builtin came from. So, if your module has an
-attribute called "``max_widgets``", you may simply use it:
-
-.. code-block:: none
-
- foo: Py_ssize_t = max_widgets
-
-If the symbol isn't found in the current module, it fails over to looking in
-``sys.modules``. That's how it can find ``sys.maxsize`` for example. (Since you
-don't know in advance what modules the user will load into their interpreter,
-it's best to restrict yourself to modules that are preloaded by Python itself.)
-
-Evaluating default values only at runtime means Argument Clinic can't compute
-the correct equivalent C default value. So you need to tell it explicitly.
-When you use an expression, you must also specify the equivalent expression
-in C, using the ``c_default`` parameter to the converter:
-
-.. code-block:: none
-
- foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1
-
-Another complication: Argument Clinic can't know in advance whether or not the
-expression you supply is valid. It parses it to make sure it looks legal, but
-it can't *actually* know. You must be very careful when using expressions to
-specify values that are guaranteed to be valid at runtime!
-
-Finally, because expressions must be representable as static C values, there
-are many restrictions on legal expressions. Here's a list of Python features
-you're not permitted to use:
-
-* Function calls.
-* Inline if statements (``3 if foo else 5``).
-* Automatic sequence unpacking (``*[1, 2, 3]``).
-* List/set/dict comprehensions and generator expressions.
-* Tuple/list/set/dict literals.
-
-
-
-Using a return converter
-------------------------
-
-By default the impl function Argument Clinic generates for you returns ``PyObject *``.
-But your C function often computes some C type, then converts it into the ``PyObject *``
-at the last moment. Argument Clinic handles converting your inputs from Python types
-into native C types—why not have it convert your return value from a native C type
-into a Python type too?
-
-That's what a "return converter" does. It changes your impl function to return
-some C type, then adds code to the generated (non-impl) function to handle converting
-that value into the appropriate ``PyObject *``.
-
-The syntax for return converters is similar to that of parameter converters.
-You specify the return converter like it was a return annotation on the
-function itself. Return converters behave much the same as parameter converters;
-they take arguments, the arguments are all keyword-only, and if you're not changing
-any of the default arguments you can omit the parentheses.
-
-(If you use both ``"as"`` *and* a return converter for your function,
-the ``"as"`` should come before the return converter.)
-
-There's one additional complication when using return converters: how do you
-indicate an error has occurred? Normally, a function returns a valid (non-``NULL``)
-pointer for success, and ``NULL`` for failure. But if you use an integer return converter,
-all integers are valid. How can Argument Clinic detect an error? Its solution: each return
-converter implicitly looks for a special value that indicates an error. If you return
-that value, and an error has been set (``PyErr_Occurred()`` returns a true
-value), then the generated code will propagate the error. Otherwise it will
-encode the value you return like normal.
-
-Currently Argument Clinic supports only a few return converters:
-
-.. code-block:: none
-
- bool
- int
- unsigned int
- long
- unsigned int
- size_t
- Py_ssize_t
- float
- double
- DecodeFSDefault
-
-None of these take parameters. For the first three, return -1 to indicate
-error. For ``DecodeFSDefault``, the return type is ``const char *``; return a ``NULL``
-pointer to indicate an error.
-
-To see all the return converters Argument Clinic supports, along with
-their parameters (if any),
-just run ``Tools/clinic/clinic.py --converters`` for the full list.
-
-
-Cloning existing functions
---------------------------
-
-If you have a number of functions that look similar, you may be able to
-use Clinic's "clone" feature. When you clone an existing function,
-you reuse:
-
-* its parameters, including
-
- * their names,
-
- * their converters, with all parameters,
-
- * their default values,
-
- * their per-parameter docstrings,
-
- * their *kind* (whether they're positional only,
- positional or keyword, or keyword only), and
-
-* its return converter.
-
-The only thing not copied from the original function is its docstring;
-the syntax allows you to specify a new docstring.
-
-Here's the syntax for cloning a function::
-
- /*[clinic input]
- module.class.new_function [as c_basename] = module.class.existing_function
-
- Docstring for new_function goes here.
- [clinic start generated code]*/
-
-(The functions can be in different modules or classes. I wrote
-``module.class`` in the sample just to illustrate that you must
-use the full path to *both* functions.)
-
-Sorry, there's no syntax for partially cloning a function, or cloning a function
-then modifying it. Cloning is an all-or nothing proposition.
-
-Also, the function you are cloning from must have been previously defined
-in the current file.
-
-Calling Python code
--------------------
-
-The rest of the advanced topics require you to write Python code
-which lives inside your C file and modifies Argument Clinic's
-runtime state. This is simple: you simply define a Python block.
-
-A Python block uses different delimiter lines than an Argument
-Clinic function block. It looks like this::
-
- /*[python input]
- # python code goes here
- [python start generated code]*/
-
-All the code inside the Python block is executed at the
-time it's parsed. All text written to stdout inside the block
-is redirected into the "output" after the block.
-
-As an example, here's a Python block that adds a static integer
-variable to the C code::
-
- /*[python input]
- print('static int __ignored_unused_variable__ = 0;')
- [python start generated code]*/
- static int __ignored_unused_variable__ = 0;
- /*[python checksum:...]*/
-
-
-Using a "self converter"
-------------------------
-
-Argument Clinic automatically adds a "self" parameter for you
-using a default converter. It automatically sets the ``type``
-of this parameter to the "pointer to an instance" you specified
-when you declared the type. However, you can override
-Argument Clinic's converter and specify one yourself.
-Just add your own ``self`` parameter as the first parameter in a
-block, and ensure that its converter is an instance of
-``self_converter`` or a subclass thereof.
-
-What's the point? This lets you override the type of ``self``,
-or give it a different default name.
-
-How do you specify the custom type you want to cast ``self`` to?
-If you only have one or two functions with the same type for ``self``,
-you can directly use Argument Clinic's existing ``self`` converter,
-passing in the type you want to use as the ``type`` parameter::
-
- /*[clinic input]
-
- _pickle.Pickler.dump
-
- self: self(type="PicklerObject *")
- obj: object
- /
-
- Write a pickled representation of the given object to the open file.
- [clinic start generated code]*/
-
-On the other hand, if you have a lot of functions that will use the same
-type for ``self``, it's best to create your own converter, subclassing
-``self_converter`` but overwriting the ``type`` member::
-
- /*[python input]
- class PicklerObject_converter(self_converter):
- type = "PicklerObject *"
- [python start generated code]*/
-
- /*[clinic input]
-
- _pickle.Pickler.dump
-
- self: PicklerObject
- obj: object
- /
-
- Write a pickled representation of the given object to the open file.
- [clinic start generated code]*/
-
-
-Using a "defining class" converter
-----------------------------------
-
-Argument Clinic facilitates gaining access to the defining class of a method.
-This is useful for :ref:`heap type ` methods that need to fetch
-module level state. Use :c:func:`PyType_FromModuleAndSpec` to associate a new
-heap type with a module. You can now use :c:func:`PyType_GetModuleState` on
-the defining class to fetch the module state, for example from a module method.
-
-Example from ``Modules/zlibmodule.c``. First, ``defining_class`` is added to
-the clinic input::
-
- /*[clinic input]
- zlib.Compress.compress
-
- cls: defining_class
- data: Py_buffer
- Binary data to be compressed.
- /
-
-
-After running the Argument Clinic tool, the following function signature is
-generated::
-
- /*[clinic start generated code]*/
- static PyObject *
- zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls,
- Py_buffer *data)
- /*[clinic end generated code: output=6731b3f0ff357ca6 input=04d00f65ab01d260]*/
-
-
-The following code can now use ``PyType_GetModuleState(cls)`` to fetch the
-module state::
-
- zlibstate *state = PyType_GetModuleState(cls);
-
-
-Each method may only have one argument using this converter, and it must appear
-after ``self``, or, if ``self`` is not used, as the first argument. The argument
-will be of type ``PyTypeObject *``. The argument will not appear in the
-``__text_signature__``.
-
-The ``defining_class`` converter is not compatible with ``__init__`` and ``__new__``
-methods, which cannot use the ``METH_METHOD`` convention.
-
-It is not possible to use ``defining_class`` with slot methods. In order to
-fetch the module state from such methods, use :c:func:`PyType_GetModuleByDef`
-to look up the module and then :c:func:`PyModule_GetState` to fetch the module
-state. Example from the ``setattro`` slot method in
-``Modules/_threadmodule.c``::
-
- static int
- local_setattro(localobject *self, PyObject *name, PyObject *v)
- {
- PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &thread_module);
- thread_module_state *state = get_thread_state(module);
- ...
- }
-
-
-See also :pep:`573`.
-
-
-Writing a custom converter
---------------------------
-
-As we hinted at in the previous section... you can write your own converters!
-A converter is simply a Python class that inherits from ``CConverter``.
-The main purpose of a custom converter is if you have a parameter using
-the ``O&`` format unit—parsing this parameter means calling
-a :c:func:`PyArg_ParseTuple` "converter function".
-
-Your converter class should be named ``*something*_converter``.
-If the name follows this convention, then your converter class
-will be automatically registered with Argument Clinic; its name
-will be the name of your class with the ``_converter`` suffix
-stripped off. (This is accomplished with a metaclass.)
-
-You shouldn't subclass ``CConverter.__init__``. Instead, you should
-write a ``converter_init()`` function. ``converter_init()``
-always accepts a ``self`` parameter; after that, all additional
-parameters *must* be keyword-only. Any arguments passed in to
-the converter in Argument Clinic will be passed along to your
-``converter_init()``.
-
-There are some additional members of ``CConverter`` you may wish
-to specify in your subclass. Here's the current list:
-
-``type``
- The C type to use for this variable.
- ``type`` should be a Python string specifying the type, e.g. ``int``.
- If this is a pointer type, the type string should end with ``' *'``.
-
-``default``
- The Python default value for this parameter, as a Python value.
- Or the magic value ``unspecified`` if there is no default.
-
-``py_default``
- ``default`` as it should appear in Python code,
- as a string.
- Or ``None`` if there is no default.
-
-``c_default``
- ``default`` as it should appear in C code,
- as a string.
- Or ``None`` if there is no default.
-
-``c_ignored_default``
- The default value used to initialize the C variable when
- there is no default, but not specifying a default may
- result in an "uninitialized variable" warning. This can
- easily happen when using option groups—although
- properly written code will never actually use this value,
- the variable does get passed in to the impl, and the
- C compiler will complain about the "use" of the
- uninitialized value. This value should always be a
- non-empty string.
-
-``converter``
- The name of the C converter function, as a string.
-
-``impl_by_reference``
- A boolean value. If true,
- Argument Clinic will add a ``&`` in front of the name of
- the variable when passing it into the impl function.
-
-``parse_by_reference``
- A boolean value. If true,
- Argument Clinic will add a ``&`` in front of the name of
- the variable when passing it into :c:func:`PyArg_ParseTuple`.
-
-
-Here's the simplest example of a custom converter, from ``Modules/zlibmodule.c``::
-
- /*[python input]
-
- class ssize_t_converter(CConverter):
- type = 'Py_ssize_t'
- converter = 'ssize_t_converter'
-
- [python start generated code]*/
- /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/
-
-This block adds a converter to Argument Clinic named ``ssize_t``. Parameters
-declared as ``ssize_t`` will be declared as type :c:type:`Py_ssize_t`, and will
-be parsed by the ``'O&'`` format unit, which will call the
-``ssize_t_converter`` converter function. ``ssize_t`` variables
-automatically support default values.
-
-More sophisticated custom converters can insert custom C code to
-handle initialization and cleanup.
-You can see more examples of custom converters in the CPython
-source tree; grep the C files for the string ``CConverter``.
-
-Writing a custom return converter
----------------------------------
-
-Writing a custom return converter is much like writing
-a custom converter. Except it's somewhat simpler, because return
-converters are themselves much simpler.
-
-Return converters must subclass ``CReturnConverter``.
-There are no examples yet of custom return converters,
-because they are not widely used yet. If you wish to
-write your own return converter, please read ``Tools/clinic/clinic.py``,
-specifically the implementation of ``CReturnConverter`` and
-all its subclasses.
-
-METH_O and METH_NOARGS
-----------------------------------------------
-
-To convert a function using ``METH_O``, make sure the function's
-single argument is using the ``object`` converter, and mark the
-arguments as positional-only::
-
- /*[clinic input]
- meth_o_sample
-
- argument: object
- /
- [clinic start generated code]*/
-
-
-To convert a function using ``METH_NOARGS``, just don't specify
-any arguments.
-
-You can still use a self converter, a return converter, and specify
-a ``type`` argument to the object converter for ``METH_O``.
-
-tp_new and tp_init functions
-----------------------------------------------
-
-You can convert ``tp_new`` and ``tp_init`` functions. Just name
-them ``__new__`` or ``__init__`` as appropriate. Notes:
-
-* The function name generated for ``__new__`` doesn't end in ``__new__``
- like it would by default. It's just the name of the class, converted
- into a valid C identifier.
-
-* No ``PyMethodDef`` ``#define`` is generated for these functions.
-
-* ``__init__`` functions return ``int``, not ``PyObject *``.
-
-* Use the docstring as the class docstring.
-
-* Although ``__new__`` and ``__init__`` functions must always
- accept both the ``args`` and ``kwargs`` objects, when converting
- you may specify any signature for these functions that you like.
- (If your function doesn't support keywords, the parsing function
- generated will throw an exception if it receives any.)
-
-Changing and redirecting Clinic's output
-----------------------------------------
-
-It can be inconvenient to have Clinic's output interspersed with
-your conventional hand-edited C code. Luckily, Clinic is configurable:
-you can buffer up its output for printing later (or earlier!), or write
-its output to a separate file. You can also add a prefix or suffix to
-every line of Clinic's generated output.
-
-While changing Clinic's output in this manner can be a boon to readability,
-it may result in Clinic code using types before they are defined, or
-your code attempting to use Clinic-generated code before it is defined.
-These problems can be easily solved by rearranging the declarations in your file,
-or moving where Clinic's generated code goes. (This is why the default behavior
-of Clinic is to output everything into the current block; while many people
-consider this hampers readability, it will never require rearranging your
-code to fix definition-before-use problems.)
-
-Let's start with defining some terminology:
-
-*field*
- A field, in this context, is a subsection of Clinic's output.
- For example, the ``#define`` for the ``PyMethodDef`` structure
- is a field, called ``methoddef_define``. Clinic has seven
- different fields it can output per function definition:
-
- .. code-block:: none
-
- docstring_prototype
- docstring_definition
- methoddef_define
- impl_prototype
- parser_prototype
- parser_definition
- impl_definition
-
- All the names are of the form ``"_"``,
- where ``""`` is the semantic object represented (the parsing function,
- the impl function, the docstring, or the methoddef structure) and ``""``
- represents what kind of statement the field is. Field names that end in
- ``"_prototype"``
- represent forward declarations of that thing, without the actual body/data
- of the thing; field names that end in ``"_definition"`` represent the actual
- definition of the thing, with the body/data of the thing. (``"methoddef"``
- is special, it's the only one that ends with ``"_define"``, representing that
- it's a preprocessor #define.)
-
-*destination*
- A destination is a place Clinic can write output to. There are
- five built-in destinations:
-
- ``block``
- The default destination: printed in the output section of
- the current Clinic block.
-
- ``buffer``
- A text buffer where you can save text for later. Text sent
- here is appended to the end of any existing text. It's an
- error to have any text left in the buffer when Clinic finishes
- processing a file.
-
- ``file``
- A separate "clinic file" that will be created automatically by Clinic.
- The filename chosen for the file is ``{basename}.clinic{extension}``,
- where ``basename`` and ``extension`` were assigned the output
- from ``os.path.splitext()`` run on the current file. (Example:
- the ``file`` destination for ``_pickle.c`` would be written to
- ``_pickle.clinic.c``.)
-
- **Important: When using a** ``file`` **destination, you**
- *must check in* **the generated file!**
-
- ``two-pass``
- A buffer like ``buffer``. However, a two-pass buffer can only
- be dumped once, and it prints out all text sent to it during
- all processing, even from Clinic blocks *after* the dumping point.
-
- ``suppress``
- The text is suppressed—thrown away.
-
-
-Clinic defines five new directives that let you reconfigure its output.
-
-The first new directive is ``dump``:
-
-.. code-block:: none
-
- dump
-
-This dumps the current contents of the named destination into the output of
-the current block, and empties it. This only works with ``buffer`` and
-``two-pass`` destinations.
-
-The second new directive is ``output``. The most basic form of ``output``
-is like this:
-
-.. code-block:: none
-
- output
-
-This tells Clinic to output *field* to *destination*. ``output`` also
-supports a special meta-destination, called ``everything``, which tells
-Clinic to output *all* fields to that *destination*.
-
-``output`` has a number of other functions:
-
-.. code-block:: none
-
- output push
- output pop
- output preset
-
-
-``output push`` and ``output pop`` allow you to push and pop
-configurations on an internal configuration stack, so that you
-can temporarily modify the output configuration, then easily restore
-the previous configuration. Simply push before your change to save
-the current configuration, then pop when you wish to restore the
-previous configuration.
-
-``output preset`` sets Clinic's output to one of several built-in
-preset configurations, as follows:
-
- ``block``
- Clinic's original starting configuration. Writes everything
- immediately after the input block.
-
- Suppress the ``parser_prototype``
- and ``docstring_prototype``, write everything else to ``block``.
-
- ``file``
- Designed to write everything to the "clinic file" that it can.
- You then ``#include`` this file near the top of your file.
- You may need to rearrange your file to make this work, though
- usually this just means creating forward declarations for various
- ``typedef`` and ``PyTypeObject`` definitions.
-
- Suppress the ``parser_prototype``
- and ``docstring_prototype``, write the ``impl_definition`` to
- ``block``, and write everything else to ``file``.
-
- The default filename is ``"{dirname}/clinic/{basename}.h"``.
-
- ``buffer``
- Save up most of the output from Clinic, to be written into
- your file near the end. For Python files implementing modules
- or builtin types, it's recommended that you dump the buffer
- just above the static structures for your module or
- builtin type; these are normally very near the end. Using
- ``buffer`` may require even more editing than ``file``, if
- your file has static ``PyMethodDef`` arrays defined in the
- middle of the file.
-
- Suppress the ``parser_prototype``, ``impl_prototype``,
- and ``docstring_prototype``, write the ``impl_definition`` to
- ``block``, and write everything else to ``file``.
-
- ``two-pass``
- Similar to the ``buffer`` preset, but writes forward declarations to
- the ``two-pass`` buffer, and definitions to the ``buffer``.
- This is similar to the ``buffer`` preset, but may require
- less editing than ``buffer``. Dump the ``two-pass`` buffer
- near the top of your file, and dump the ``buffer`` near
- the end just like you would when using the ``buffer`` preset.
-
- Suppresses the ``impl_prototype``, write the ``impl_definition``
- to ``block``, write ``docstring_prototype``, ``methoddef_define``,
- and ``parser_prototype`` to ``two-pass``, write everything else
- to ``buffer``.
-
- ``partial-buffer``
- Similar to the ``buffer`` preset, but writes more things to ``block``,
- only writing the really big chunks of generated code to ``buffer``.
- This avoids the definition-before-use problem of ``buffer`` completely,
- at the small cost of having slightly more stuff in the block's output.
- Dump the ``buffer`` near the end, just like you would when using
- the ``buffer`` preset.
-
- Suppresses the ``impl_prototype``, write the ``docstring_definition``
- and ``parser_definition`` to ``buffer``, write everything else to ``block``.
-
-The third new directive is ``destination``:
-
-.. code-block:: none
-
- destination [...]
-
-This performs an operation on the destination named ``name``.
-
-There are two defined subcommands: ``new`` and ``clear``.
-
-The ``new`` subcommand works like this:
-
-.. code-block:: none
-
- destination new
-
-This creates a new destination with name ```` and type ````.
-
-There are five destination types:
-
- ``suppress``
- Throws the text away.
-
- ``block``
- Writes the text to the current block. This is what Clinic
- originally did.
-
- ``buffer``
- A simple text buffer, like the "buffer" builtin destination above.
-
- ``file``
- A text file. The file destination takes an extra argument,
- a template to use for building the filename, like so:
-
- destination new
-
- The template can use three strings internally that will be replaced
- by bits of the filename:
-
- {path}
- The full path to the file, including directory and full filename.
- {dirname}
- The name of the directory the file is in.
- {basename}
- Just the name of the file, not including the directory.
- {basename_root}
- Basename with the extension clipped off
- (everything up to but not including the last '.').
- {basename_extension}
- The last '.' and everything after it. If the basename
- does not contain a period, this will be the empty string.
-
- If there are no periods in the filename, {basename} and {filename}
- are the same, and {extension} is empty. "{basename}{extension}"
- is always exactly the same as "{filename}"."
-
- ``two-pass``
- A two-pass buffer, like the "two-pass" builtin destination above.
-
-
-The ``clear`` subcommand works like this:
-
-.. code-block:: none
-
- destination clear
-
-It removes all the accumulated text up to this point in the destination.
-(I don't know what you'd need this for, but I thought maybe it'd be
-useful while someone's experimenting.)
-
-The fourth new directive is ``set``:
-
-.. code-block:: none
-
- set line_prefix "string"
- set line_suffix "string"
-
-``set`` lets you set two internal variables in Clinic.
-``line_prefix`` is a string that will be prepended to every line of Clinic's output;
-``line_suffix`` is a string that will be appended to every line of Clinic's output.
-
-Both of these support two format strings:
-
- ``{block comment start}``
- Turns into the string ``/*``, the start-comment text sequence for C files.
-
- ``{block comment end}``
- Turns into the string ``*/``, the end-comment text sequence for C files.
-
-The final new directive is one you shouldn't need to use directly,
-called ``preserve``:
-
-.. code-block:: none
-
- preserve
-
-This tells Clinic that the current contents of the output should be kept, unmodified.
-This is used internally by Clinic when dumping output into ``file`` files; wrapping
-it in a Clinic block lets Clinic use its existing checksum functionality to ensure
-the file was not modified by hand before it gets overwritten.
-
-
-The #ifdef trick
-----------------------------------------------
-
-If you're converting a function that isn't available on all platforms,
-there's a trick you can use to make life a little easier. The existing
-code probably looks like this::
-
- #ifdef HAVE_FUNCTIONNAME
- static module_functionname(...)
- {
- ...
- }
- #endif /* HAVE_FUNCTIONNAME */
-
-And then in the ``PyMethodDef`` structure at the bottom the existing code
-will have:
-
-.. code-block:: none
-
- #ifdef HAVE_FUNCTIONNAME
- {'functionname', ... },
- #endif /* HAVE_FUNCTIONNAME */
-
-In this scenario, you should enclose the body of your impl function inside the ``#ifdef``,
-like so::
-
- #ifdef HAVE_FUNCTIONNAME
- /*[clinic input]
- module.functionname
- ...
- [clinic start generated code]*/
- static module_functionname(...)
- {
- ...
- }
- #endif /* HAVE_FUNCTIONNAME */
-
-Then, remove those three lines from the ``PyMethodDef`` structure,
-replacing them with the macro Argument Clinic generated:
-
-.. code-block:: none
-
- MODULE_FUNCTIONNAME_METHODDEF
-
-(You can find the real name for this macro inside the generated code.
-Or you can calculate it yourself: it's the name of your function as defined
-on the first line of your block, but with periods changed to underscores,
-uppercased, and ``"_METHODDEF"`` added to the end.)
-
-Perhaps you're wondering: what if ``HAVE_FUNCTIONNAME`` isn't defined?
-The ``MODULE_FUNCTIONNAME_METHODDEF`` macro won't be defined either!
-
-Here's where Argument Clinic gets very clever. It actually detects that the
-Argument Clinic block might be deactivated by the ``#ifdef``. When that
-happens, it generates a little extra code that looks like this::
-
- #ifndef MODULE_FUNCTIONNAME_METHODDEF
- #define MODULE_FUNCTIONNAME_METHODDEF
- #endif /* !defined(MODULE_FUNCTIONNAME_METHODDEF) */
-
-That means the macro always works. If the function is defined, this turns
-into the correct structure, including the trailing comma. If the function is
-undefined, this turns into nothing.
-
-However, this causes one ticklish problem: where should Argument Clinic put this
-extra code when using the "block" output preset? It can't go in the output block,
-because that could be deactivated by the ``#ifdef``. (That's the whole point!)
-
-In this situation, Argument Clinic writes the extra code to the "buffer" destination.
-This may mean that you get a complaint from Argument Clinic:
-
-.. code-block:: none
-
- Warning in file "Modules/posixmodule.c" on line 12357:
- Destination buffer 'buffer' not empty at end of file, emptying.
-
-When this happens, just open your file, find the ``dump buffer`` block that
-Argument Clinic added to your file (it'll be at the very bottom), then
-move it above the ``PyMethodDef`` structure where that macro is used.
-
-
-
-Using Argument Clinic in Python files
--------------------------------------
-
-It's actually possible to use Argument Clinic to preprocess Python files.
-There's no point to using Argument Clinic blocks, of course, as the output
-wouldn't make any sense to the Python interpreter. But using Argument Clinic
-to run Python blocks lets you use Python as a Python preprocessor!
-
-Since Python comments are different from C comments, Argument Clinic
-blocks embedded in Python files look slightly different. They look like this:
-
-.. code-block:: python3
-
- #/*[python input]
- #print("def foo(): pass")
- #[python start generated code]*/
- def foo(): pass
- #/*[python checksum:...]*/
+ The Argument Clinic How-TO has been moved to the `Python Developer's Guide
+ `__.
diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst
index 83d80471ffc8ee0..4828e2fa29bd247 100644
--- a/Doc/howto/curses.rst
+++ b/Doc/howto/curses.rst
@@ -4,6 +4,8 @@
Curses Programming with Python
**********************************
+.. currentmodule:: curses
+
:Author: A.M. Kuchling, Eric S. Raymond
:Release: 2.04
@@ -65,7 +67,7 @@ The Python module is a fairly simple wrapper over the C functions provided by
curses; if you're already familiar with curses programming in C, it's really
easy to transfer that knowledge to Python. The biggest difference is that the
Python interface makes things simpler by merging different C functions such as
-:c:func:`addstr`, :c:func:`mvaddstr`, and :c:func:`mvwaddstr` into a single
+:c:func:`!addstr`, :c:func:`!mvaddstr`, and :c:func:`!mvwaddstr` into a single
:meth:`~curses.window.addstr` method. You'll see this covered in more
detail later.
@@ -82,7 +84,7 @@ Before doing anything, curses must be initialized. This is done by
calling the :func:`~curses.initscr` function, which will determine the
terminal type, send any required setup codes to the terminal, and
create various internal data structures. If successful,
-:func:`initscr` returns a window object representing the entire
+:func:`!initscr` returns a window object representing the entire
screen; this is usually called ``stdscr`` after the name of the
corresponding C variable. ::
@@ -151,8 +153,8 @@ importing the :func:`curses.wrapper` function and using it like this::
The :func:`~curses.wrapper` function takes a callable object and does the
initializations described above, also initializing colors if color
-support is present. :func:`wrapper` then runs your provided callable.
-Once the callable returns, :func:`wrapper` will restore the original
+support is present. :func:`!wrapper` then runs your provided callable.
+Once the callable returns, :func:`!wrapper` will restore the original
state of the terminal. The callable is called inside a
:keyword:`try`...\ :keyword:`except` that catches exceptions, restores
the state of the terminal, and then re-raises the exception. Therefore
@@ -200,7 +202,7 @@ This is because curses was originally written with slow 300-baud
terminal connections in mind; with these terminals, minimizing the
time required to redraw the screen was very important. Instead curses
accumulates changes to the screen and displays them in the most
-efficient manner when you call :meth:`refresh`. For example, if your
+efficient manner when you call :meth:`!refresh`. For example, if your
program displays some text in a window and then clears the window,
there's no need to send the original text because they're never
visible.
@@ -210,7 +212,7 @@ really complicate programming with curses much. Most programs go into a flurry
of activity, and then pause waiting for a keypress or some other action on the
part of the user. All you have to do is to be sure that the screen has been
redrawn before pausing to wait for user input, by first calling
-``stdscr.refresh()`` or the :meth:`refresh` method of some other relevant
+:meth:`!stdscr.refresh` or the :meth:`!refresh` method of some other relevant
window.
A pad is a special case of a window; it can be larger than the actual display
@@ -234,7 +236,7 @@ displayed. ::
# : filled with pad content.
pad.refresh( 0,0, 5,5, 20,75)
-The :meth:`refresh` call displays a section of the pad in the rectangle
+The :meth:`!refresh` call displays a section of the pad in the rectangle
extending from coordinate (5,5) to coordinate (20,75) on the screen; the upper
left corner of the displayed section is coordinate (0,0) on the pad. Beyond
that difference, pads are exactly like ordinary windows and support the same
@@ -242,7 +244,7 @@ methods.
If you have multiple windows and pads on screen there is a more
efficient way to update the screen and prevent annoying screen flicker
-as each part of the screen gets updated. :meth:`refresh` actually
+as each part of the screen gets updated. :meth:`!refresh` actually
does two things:
1) Calls the :meth:`~curses.window.noutrefresh` method of each window
@@ -251,8 +253,8 @@ does two things:
2) Calls the function :func:`~curses.doupdate` function to change the
physical screen to match the desired state recorded in the data structure.
-Instead you can call :meth:`noutrefresh` on a number of windows to
-update the data structure, and then call :func:`doupdate` to update
+Instead you can call :meth:`!noutrefresh` on a number of windows to
+update the data structure, and then call :func:`!doupdate` to update
the screen.
@@ -261,11 +263,11 @@ Displaying Text
From a C programmer's point of view, curses may sometimes look like a
twisty maze of functions, all subtly different. For example,
-:c:func:`addstr` displays a string at the current cursor location in
-the ``stdscr`` window, while :c:func:`mvaddstr` moves to a given y,x
-coordinate first before displaying the string. :c:func:`waddstr` is just
-like :c:func:`addstr`, but allows specifying a window to use instead of
-using ``stdscr`` by default. :c:func:`mvwaddstr` allows specifying both
+:c:func:`!addstr` displays a string at the current cursor location in
+the ``stdscr`` window, while :c:func:`!mvaddstr` moves to a given y,x
+coordinate first before displaying the string. :c:func:`!waddstr` is just
+like :c:func:`!addstr`, but allows specifying a window to use instead of
+using ``stdscr`` by default. :c:func:`!mvwaddstr` allows specifying both
a window and a coordinate.
Fortunately the Python interface hides all these details. ``stdscr``
@@ -298,7 +300,7 @@ the next subsection.
The :meth:`~curses.window.addstr` method takes a Python string or
bytestring as the value to be displayed. The contents of bytestrings
are sent to the terminal as-is. Strings are encoded to bytes using
-the value of the window's :attr:`encoding` attribute; this defaults to
+the value of the window's :attr:`~window.encoding` attribute; this defaults to
the default system encoding as returned by :func:`locale.getencoding`.
The :meth:`~curses.window.addch` methods take a character, which can be
@@ -444,15 +446,15 @@ There are two methods for getting input from a window:
It's possible to not wait for the user using the
:meth:`~curses.window.nodelay` window method. After ``nodelay(True)``,
-:meth:`getch` and :meth:`getkey` for the window become
-non-blocking. To signal that no input is ready, :meth:`getch` returns
-``curses.ERR`` (a value of -1) and :meth:`getkey` raises an exception.
+:meth:`!getch` and :meth:`!getkey` for the window become
+non-blocking. To signal that no input is ready, :meth:`!getch` returns
+``curses.ERR`` (a value of -1) and :meth:`!getkey` raises an exception.
There's also a :func:`~curses.halfdelay` function, which can be used to (in
-effect) set a timer on each :meth:`getch`; if no input becomes
+effect) set a timer on each :meth:`!getch`; if no input becomes
available within a specified delay (measured in tenths of a second),
curses raises an exception.
-The :meth:`getch` method returns an integer; if it's between 0 and 255, it
+The :meth:`!getch` method returns an integer; if it's between 0 and 255, it
represents the ASCII code of the key pressed. Values greater than 255 are
special keys such as Page Up, Home, or the cursor keys. You can compare the
value returned to constants such as :const:`curses.KEY_PPAGE`,
@@ -525,7 +527,7 @@ If you're in doubt about the detailed behavior of the curses
functions, consult the manual pages for your curses implementation,
whether it's ncurses or a proprietary Unix vendor's. The manual pages
will document any quirks, and provide complete lists of all the
-functions, attributes, and :const:`ACS_\*` characters available to
+functions, attributes, and :ref:`ACS_\* ` characters available to
you.
Because the curses API is so large, some functions aren't supported in
diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst
index 74710d9b3fc2edc..f732aaea729d401 100644
--- a/Doc/howto/descriptor.rst
+++ b/Doc/howto/descriptor.rst
@@ -521,11 +521,11 @@ everyday Python programs.
Descriptor protocol
-------------------
-``descr.__get__(self, obj, type=None) -> value``
+``descr.__get__(self, obj, type=None)``
-``descr.__set__(self, obj, value) -> None``
+``descr.__set__(self, obj, value)``
-``descr.__delete__(self, obj) -> None``
+``descr.__delete__(self, obj)``
That is all there is to it. Define any of these methods and an object is
considered a descriptor and can override default behavior upon being looked up
@@ -779,8 +779,8 @@ by a search through the class's :term:`method resolution order`.
If a descriptor is found, it is invoked with ``desc.__get__(None, A)``.
-The full C implementation can be found in :c:func:`type_getattro()` and
-:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`.
+The full C implementation can be found in :c:func:`!type_getattro` and
+:c:func:`!_PyType_Lookup` in :source:`Objects/typeobject.c`.
Invocation from super
@@ -794,7 +794,7 @@ for the base class ``B`` immediately following ``A`` and then returns
``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned
unchanged.
-The full C implementation can be found in :c:func:`super_getattro()` in
+The full C implementation can be found in :c:func:`!super_getattro` in
:source:`Objects/typeobject.c`. A pure Python equivalent can be found in
`Guido's Tutorial
`_.
@@ -836,8 +836,8 @@ and if they define :meth:`__set_name__`, that method is called with two
arguments. The *owner* is the class where the descriptor is used, and the
*name* is the class variable the descriptor was assigned to.
-The implementation details are in :c:func:`type_new()` and
-:c:func:`set_names()` in :source:`Objects/typeobject.c`.
+The implementation details are in :c:func:`!type_new` and
+:c:func:`!set_names` in :source:`Objects/typeobject.c`.
Since the update logic is in :meth:`type.__new__`, notifications only take
place at the time of class creation. If descriptors are added to the class
@@ -943,6 +943,10 @@ it can be updated:
>>> Movie('Star Wars').director
'J.J. Abrams'
+.. testcleanup::
+
+ conn.close()
+
Pure Python Equivalents
^^^^^^^^^^^^^^^^^^^^^^^
@@ -1009,17 +1013,23 @@ here is a pure Python equivalent:
if obj is None:
return self
if self.fget is None:
- raise AttributeError(f"property '{self._name}' has no getter")
+ raise AttributeError(
+ f'property {self._name!r} of {type(obj).__name__!r} object has no getter'
+ )
return self.fget(obj)
def __set__(self, obj, value):
if self.fset is None:
- raise AttributeError(f"property '{self._name}' has no setter")
+ raise AttributeError(
+ f'property {self._name!r} of {type(obj).__name__!r} object has no setter'
+ )
self.fset(obj, value)
def __delete__(self, obj):
if self.fdel is None:
- raise AttributeError(f"property '{self._name}' has no deleter")
+ raise AttributeError(
+ f'property {self._name!r} of {type(obj).__name__!r} object has no deleter'
+ )
self.fdel(obj)
def getter(self, fget):
@@ -1050,6 +1060,11 @@ here is a pure Python equivalent:
def delx(self):
del self.__x
x = Property(getx, setx, delx, "I'm the 'x' property.")
+ no_getter = Property(None, setx, delx, "I'm the 'x' property.")
+ no_setter = Property(getx, None, delx, "I'm the 'x' property.")
+ no_deleter = Property(getx, setx, None, "I'm the 'x' property.")
+ no_doc = Property(getx, setx, delx, None)
+
# Now do it again but use the decorator style
@@ -1088,6 +1103,32 @@ here is a pure Python equivalent:
>>> hasattr(ccc, 'x')
False
+ >>> cc = CC()
+ >>> cc.x = 33
+ >>> try:
+ ... cc.no_getter
+ ... except AttributeError as e:
+ ... e.args[0]
+ ...
+ "property 'no_getter' of 'CC' object has no getter"
+
+ >>> try:
+ ... cc.no_setter = 33
+ ... except AttributeError as e:
+ ... e.args[0]
+ ...
+ "property 'no_setter' of 'CC' object has no setter"
+
+ >>> try:
+ ... del cc.no_deleter
+ ... except AttributeError as e:
+ ... e.args[0]
+ ...
+ "property 'no_deleter' of 'CC' object has no deleter"
+
+ >>> CC.no_doc.__doc__ is None
+ True
+
The :func:`property` builtin helps whenever a user interface has granted
attribute access and then subsequent changes require the intervention of a
method.
@@ -1141,6 +1182,16 @@ roughly equivalent to:
obj = self.__self__
return func(obj, *args, **kwargs)
+ def __getattribute__(self, name):
+ "Emulate method_getset() in Objects/classobject.c"
+ if name == '__doc__':
+ return self.__func__.__doc__
+ return object.__getattribute__(self, name)
+
+ def __getattr__(self, name):
+ "Emulate method_getattro() in Objects/classobject.c"
+ return getattr(self.__func__, name)
+
To support automatic creation of methods, functions include the
:meth:`__get__` method for binding methods during attribute access. This
means that functions are non-data descriptors that return bound methods
@@ -1273,11 +1324,14 @@ Using the non-data descriptor protocol, a pure Python version of
.. testcode::
+ import functools
+
class StaticMethod:
"Emulate PyStaticMethod_Type() in Objects/funcobject.c"
def __init__(self, f):
self.f = f
+ functools.update_wrapper(self, f)
def __get__(self, obj, objtype=None):
return self.f
@@ -1285,13 +1339,19 @@ Using the non-data descriptor protocol, a pure Python version of
def __call__(self, *args, **kwds):
return self.f(*args, **kwds)
+The :func:`functools.update_wrapper` call adds a ``__wrapped__`` attribute
+that refers to the underlying function. Also it carries forward
+the attributes necessary to make the wrapper look like the wrapped
+function: ``__name__``, ``__qualname__``, ``__doc__``, and ``__annotations__``.
+
.. testcode::
:hide:
class E_sim:
@StaticMethod
- def f(x):
- return x * 10
+ def f(x: int) -> str:
+ "Simple function example"
+ return "!" * x
wrapped_ord = StaticMethod(ord)
@@ -1299,11 +1359,51 @@ Using the non-data descriptor protocol, a pure Python version of
:hide:
>>> E_sim.f(3)
- 30
+ '!!!'
>>> E_sim().f(3)
- 30
+ '!!!'
+
+ >>> sm = vars(E_sim)['f']
+ >>> type(sm).__name__
+ 'StaticMethod'
+ >>> f = E_sim.f
+ >>> type(f).__name__
+ 'function'
+ >>> sm.__name__
+ 'f'
+ >>> f.__name__
+ 'f'
+ >>> sm.__qualname__
+ 'E_sim.f'
+ >>> f.__qualname__
+ 'E_sim.f'
+ >>> sm.__doc__
+ 'Simple function example'
+ >>> f.__doc__
+ 'Simple function example'
+ >>> sm.__annotations__
+ {'x': , 'return': }
+ >>> f.__annotations__
+ {'x': , 'return': }
+ >>> sm.__module__ == f.__module__
+ True
+ >>> sm(3)
+ '!!!'
+ >>> f(3)
+ '!!!'
+
>>> wrapped_ord('A')
65
+ >>> wrapped_ord.__module__ == ord.__module__
+ True
+ >>> wrapped_ord.__wrapped__ == ord
+ True
+ >>> wrapped_ord.__name__ == ord.__name__
+ True
+ >>> wrapped_ord.__qualname__ == ord.__qualname__
+ True
+ >>> wrapped_ord.__doc__ == ord.__doc__
+ True
Class methods
@@ -1359,19 +1459,18 @@ Using the non-data descriptor protocol, a pure Python version of
.. testcode::
+ import functools
+
class ClassMethod:
"Emulate PyClassMethod_Type() in Objects/funcobject.c"
def __init__(self, f):
self.f = f
+ functools.update_wrapper(self, f)
def __get__(self, obj, cls=None):
if cls is None:
cls = type(obj)
- if hasattr(type(self.f), '__get__'):
- # This code path was added in Python 3.9
- # and was deprecated in Python 3.11.
- return self.f.__get__(cls, cls)
return MethodType(self.f, cls)
.. testcode::
@@ -1380,48 +1479,51 @@ Using the non-data descriptor protocol, a pure Python version of
# Verify the emulation works
class T:
@ClassMethod
- def cm(cls, x, y):
- return (cls, x, y)
-
- @ClassMethod
- @property
- def __doc__(cls):
- return f'A doc for {cls.__name__!r}'
+ def cm(cls, x: int, y: str) -> tuple[str, int, str]:
+ "Class method that returns a tuple"
+ return (cls.__name__, x, y)
.. doctest::
:hide:
>>> T.cm(11, 22)
- (, 11, 22)
+ ('T', 11, 22)
# Also call it from an instance
>>> t = T()
>>> t.cm(11, 22)
- (, 11, 22)
-
- # Check the alternate path for chained descriptors
- >>> T.__doc__
- "A doc for 'T'"
-
-
-The code path for ``hasattr(type(self.f), '__get__')`` was added in
-Python 3.9 and makes it possible for :func:`classmethod` to support
-chained decorators. For example, a classmethod and property could be
-chained together. In Python 3.11, this functionality was deprecated.
-
-.. testcode::
-
- class G:
- @classmethod
- @property
- def __doc__(cls):
- return f'A doc for {cls.__name__!r}'
-
-.. doctest::
-
- >>> G.__doc__
- "A doc for 'G'"
+ ('T', 11, 22)
+
+ # Verify that T uses our emulation
+ >>> type(vars(T)['cm']).__name__
+ 'ClassMethod'
+
+ # Verify that update_wrapper() correctly copied attributes
+ >>> T.cm.__name__
+ 'cm'
+ >>> T.cm.__qualname__
+ 'T.cm'
+ >>> T.cm.__doc__
+ 'Class method that returns a tuple'
+ >>> T.cm.__annotations__
+ {'x': , 'y': , 'return': tuple[str, int, str]}
+
+ # Verify that __wrapped__ was added and works correctly
+ >>> f = vars(T)['cm'].__wrapped__
+ >>> type(f).__name__
+ 'function'
+ >>> f.__name__
+ 'cm'
+ >>> f(T, 11, 22)
+ ('T', 11, 22)
+
+
+The :func:`functools.update_wrapper` call in ``ClassMethod`` adds a
+``__wrapped__`` attribute that refers to the underlying function. Also
+it carries forward the attributes necessary to make the wrapper look
+like the wrapped function: ``__name__``, ``__qualname__``, ``__doc__``,
+and ``__annotations__``.
Member objects and __slots__
diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst
index 03f05657997cfda..a136c76303c8ef7 100644
--- a/Doc/howto/enum.rst
+++ b/Doc/howto/enum.rst
@@ -36,8 +36,10 @@ inherits from :class:`Enum` itself.
.. note:: Case of Enum Members
- Because Enums are used to represent constants we recommend using
- UPPER_CASE names for members, and will be using that style in our examples.
+ Because Enums are used to represent constants, and to help avoid issues
+ with name clashes between mixin-class methods/attributes and enum names,
+ we strongly recommend using UPPER_CASE names for members, and will be using
+ that style in our examples.
Depending on the nature of the enum a member's value may or may not be
important, but either way that value can be used to get the corresponding
@@ -158,6 +160,7 @@ And a function to display the chores for a given day::
... for chore, days in chores.items():
... if day in days:
... print(chore)
+ ...
>>> show_chores(chores_for_ethan, Weekday.SATURDAY)
answer SO questions
@@ -173,6 +176,7 @@ yourself some work and use :func:`auto()` for the values::
... FRIDAY = auto()
... SATURDAY = auto()
... SUNDAY = auto()
+ ... WEEKEND = SATURDAY | SUNDAY
.. _enum-advanced-tutorial:
@@ -282,6 +286,7 @@ The values are chosen by :func:`_generate_next_value_`, which can be
overridden::
>>> class AutoName(Enum):
+ ... @staticmethod
... def _generate_next_value_(name, start, count, last_values):
... return name
...
@@ -305,6 +310,10 @@ Iterating over the members of an enum does not provide the aliases::
>>> list(Shape)
[, , ]
+ >>> list(Weekday)
+ [, , , , , , ]
+
+Note that the aliases ``Shape.ALIAS_FOR_SQUARE`` and ``Weekday.WEEKEND`` aren't shown.
The special attribute ``__members__`` is a read-only ordered mapping of names
to members. It includes all names defined in the enumeration, including the
@@ -324,6 +333,11 @@ the enumeration members. For example, finding all the aliases::
>>> [name for name, member in Shape.__members__.items() if member.name != name]
['ALIAS_FOR_SQUARE']
+.. note::
+
+ Aliases for flags include values with multiple flags set, such as ``3``,
+ and no flags set, i.e. ``0``.
+
Comparisons
-----------
@@ -361,6 +375,11 @@ below)::
>>> Color.BLUE == 2
False
+.. warning::
+
+ It is possible to reload modules -- if a reloaded module contains
+ enums, they will be recreated, and the new members may not
+ compare identical/equal to the original members.
Allowed members and attributes of enumerations
----------------------------------------------
@@ -407,10 +426,17 @@ enumeration, with the exception of special methods (:meth:`__str__`,
:meth:`__add__`, etc.), descriptors (methods are also descriptors), and
variable names listed in :attr:`_ignore_`.
-Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then
+Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__`,
any value(s) given to the enum member will be passed into those methods.
See `Planet`_ for an example.
+.. note::
+
+ The :meth:`__new__` method, if defined, is used during creation of the Enum
+ members; it is then replaced by Enum's :meth:`__new__` which is used after
+ class creation for lookup of existing members. See :ref:`new-vs-init` for
+ more details.
+
Restricted Enum subclassing
---------------------------
@@ -449,6 +475,36 @@ sense to allow sharing some common behavior between a group of enumerations.
(See `OrderedEnum`_ for an example.)
+.. _enum-dataclass-support:
+
+Dataclass support
+-----------------
+
+When inheriting from a :class:`~dataclasses.dataclass`,
+the :meth:`~Enum.__repr__` omits the inherited class' name. For example::
+
+ >>> from dataclasses import dataclass, field
+ >>> @dataclass
+ ... class CreatureDataMixin:
+ ... size: str
+ ... legs: int
+ ... tail: bool = field(repr=False, default=True)
+ ...
+ >>> class Creature(CreatureDataMixin, Enum):
+ ... BEETLE = 'small', 6
+ ... DOG = 'medium', 4
+ ...
+ >>> Creature.DOG
+
+
+Use the :func:`!dataclass` argument ``repr=False``
+to use the standard :func:`repr`.
+
+.. versionchanged:: 3.12
+ Only the dataclass fields are shown in the value area, not the dataclass'
+ name.
+
+
Pickling
--------
@@ -469,7 +525,17 @@ from that module.
nested in other classes.
It is possible to modify how enum members are pickled/unpickled by defining
-:meth:`__reduce_ex__` in the enumeration class.
+:meth:`__reduce_ex__` in the enumeration class. The default method is by-value,
+but enums with complicated values may want to use by-name::
+
+ >>> import enum
+ >>> class MyEnum(enum.Enum):
+ ... __reduce_ex__ = enum.pickle_by_enum_name
+
+.. note::
+
+ Using by-name for flags is not recommended, as unnamed aliases will
+ not unpickle.
Functional API
@@ -541,9 +607,9 @@ The complete signature is::
start=1,
)
-:value: What the new enum class will record as its name.
+* *value*: What the new enum class will record as its name.
-:names: The enum members. This can be a whitespace- or comma-separated string
+* *names*: The enum members. This can be a whitespace- or comma-separated string
(values will start at 1 unless otherwise specified)::
'RED GREEN BLUE' | 'RED,GREEN,BLUE' | 'RED, GREEN, BLUE'
@@ -560,13 +626,13 @@ The complete signature is::
{'CHARTREUSE': 7, 'SEA_GREEN': 11, 'ROSEMARY': 42}
-:module: name of module where new enum class can be found.
+* *module*: name of module where new enum class can be found.
-:qualname: where in module new enum class can be found.
+* *qualname*: where in module new enum class can be found.
-:type: type to mix in to new enum class.
+* *type*: type to mix in to new enum class.
-:start: number to start counting at if only names are passed in.
+* *start*: number to start counting at if only names are passed in.
.. versionchanged:: 3.5
The *start* parameter was added.
@@ -677,6 +743,7 @@ It is also possible to name the combinations::
... W = 2
... X = 1
... RWX = 7
+ ...
>>> Perm.RWX
>>> ~Perm.RWX
@@ -705,7 +772,7 @@ be combined with them (but may lose :class:`IntFlag` membership::
>>> Perm.X | 4
- >>> Perm.X | 8
+ >>> Perm.X + 8
9
.. note::
@@ -751,7 +818,7 @@ flags being set, the boolean evaluation is :data:`False`::
False
Individual flags should have values that are powers of two (1, 2, 4, 8, ...),
-while combinations of flags won't::
+while combinations of flags will not::
>>> class Color(Flag):
... RED = auto()
@@ -822,19 +889,23 @@ Some rules:
4. When another data type is mixed in, the :attr:`value` attribute is *not the
same* as the enum member itself, although it is equivalent and will compare
equal.
-5. %-style formatting: ``%s`` and ``%r`` call the :class:`Enum` class's
+5. A ``data type`` is a mixin that defines :meth:`__new__`, or a
+ :class:`~dataclasses.dataclass`
+6. %-style formatting: ``%s`` and ``%r`` call the :class:`Enum` class's
:meth:`__str__` and :meth:`__repr__` respectively; other codes (such as
``%i`` or ``%h`` for IntEnum) treat the enum member as its mixed-in type.
-6. :ref:`Formatted string literals `, :meth:`str.format`,
+7. :ref:`Formatted string literals `, :meth:`str.format`,
and :func:`format` will use the enum's :meth:`__str__` method.
.. note::
Because :class:`IntEnum`, :class:`IntFlag`, and :class:`StrEnum` are
designed to be drop-in replacements for existing constants, their
- :meth:`__str__` method has been reset to their data types
+ :meth:`__str__` method has been reset to their data types'
:meth:`__str__` method.
+.. _new-vs-init:
+
When to use :meth:`__new__` vs. :meth:`__init__`
------------------------------------------------
@@ -867,6 +938,11 @@ want one of them to be the value::
>>> print(Coordinate(3))
Coordinate.VY
+.. warning::
+
+ *Do not* call ``super().__new__()``, as the lookup-only ``__new__`` is the one
+ that is found; instead, use the data type directly.
+
Finer Points
^^^^^^^^^^^^
@@ -945,12 +1021,13 @@ but remain normal attributes.
""""""""""""""""""""
Enum members are instances of their enum class, and are normally accessed as
-``EnumClass.member``. In Python versions starting with ``3.5`` you could access
-members from other members -- this practice is discouraged, is deprecated
-in ``3.12``, and will be removed in ``3.14``.
+``EnumClass.member``. In certain situations, such as writing custom enum
+behavior, being able to access one member directly from another is useful,
+and is supported; however, in order to avoid name clashes between member names
+and attributes/methods from mixed-in classes, upper-case names are strongly
+recommended.
.. versionchanged:: 3.5
-.. versionchanged:: 3.12
Creating members that are mixed with other data types
@@ -1081,13 +1158,14 @@ the following are true:
There is a new boundary mechanism that controls how out-of-range / invalid
bits are handled: ``STRICT``, ``CONFORM``, ``EJECT``, and ``KEEP``:
- * STRICT --> raises an exception when presented with invalid values
- * CONFORM --> discards any invalid bits
- * EJECT --> lose Flag status and become a normal int with the given value
- * KEEP --> keep the extra bits
- - keeps Flag status and extra bits
- - extra bits do not show up in iteration
- - extra bits do show up in repr() and str()
+* STRICT --> raises an exception when presented with invalid values
+* CONFORM --> discards any invalid bits
+* EJECT --> lose Flag status and become a normal int with the given value
+* KEEP --> keep the extra bits
+
+ - keeps Flag status and extra bits
+ - extra bits do not show up in iteration
+ - extra bits do show up in repr() and str()
The default for Flag is ``STRICT``, the default for ``IntFlag`` is ``EJECT``,
and the default for ``_convert_`` is ``KEEP`` (see ``ssl.Options`` for an
@@ -1096,8 +1174,8 @@ example of when ``KEEP`` is needed).
.. _enum-class-differences:
-How are Enums different?
-------------------------
+How are Enums and Flags different?
+----------------------------------
Enums have a custom metaclass that affects many aspects of both derived :class:`Enum`
classes and their instances (members).
@@ -1114,6 +1192,13 @@ responsible for ensuring that various other methods on the final :class:`Enum`
class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`,
:meth:`__str__` and :meth:`__repr__`).
+Flag Classes
+^^^^^^^^^^^^
+
+Flags have an expanded view of aliasing: to be canonical, the value of a flag
+needs to be a power-of-two value, and not a duplicate name. So, in addition to the
+:class:`Enum` definition of alias, a flag with no value (a.k.a. ``0``) or with more than one
+power-of-two value (e.g. ``3``) is considered an alias.
Enum Members (aka instances)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1123,9 +1208,35 @@ The most interesting thing about enum members is that they are singletons.
and then puts a custom :meth:`__new__` in place to ensure that no new ones are
ever instantiated by returning only the existing member instances.
+Flag Members
+^^^^^^^^^^^^
+
+Flag members can be iterated over just like the :class:`Flag` class, and only the
+canonical members will be returned. For example::
+
+ >>> list(Color)
+ [, , ]
+
+(Note that ``BLACK``, ``PURPLE``, and ``WHITE`` do not show up.)
+
+Inverting a flag member returns the corresponding positive value,
+rather than a negative value --- for example::
+
+ >>> ~Color.RED
+
+
+Flag members have a length corresponding to the number of power-of-two values
+they contain. For example::
+
+ >>> len(Color.PURPLE)
+ 2
+
.. _enum-cookbook:
+Enum Cookbook
+-------------
+
While :class:`Enum`, :class:`IntEnum`, :class:`StrEnum`, :class:`Flag`, and
:class:`IntFlag` are expected to cover the majority of use-cases, they cannot
@@ -1259,6 +1370,13 @@ to handle any extra arguments::
members; it is then replaced by Enum's :meth:`__new__` which is used after
class creation for lookup of existing members.
+.. warning::
+
+ *Do not* call ``super().__new__()``, as the lookup-only ``__new__`` is the one
+ that is found; instead, use the data type directly -- e.g.::
+
+ obj = int.__new__(cls, value)
+
OrderedEnum
^^^^^^^^^^^
@@ -1299,7 +1417,7 @@ enumerations)::
DuplicateFreeEnum
^^^^^^^^^^^^^^^^^
-Raises an error if a duplicate member name is found instead of creating an
+Raises an error if a duplicate member value is found instead of creating an
alias::
>>> class DuplicateFreeEnum(Enum):
@@ -1319,8 +1437,9 @@ alias::
... GRENE = 2
...
Traceback (most recent call last):
- ...
+ ...
ValueError: aliases not allowed in DuplicateFreeEnum: 'GRENE' --> 'GREEN'
+ Error calling __set_name__ on '_proto_member' instance 'GRENE' in 'Color'
.. note::
diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst
index 38a651b0f964a6b..b0f9d22d74f0e31 100644
--- a/Doc/howto/functional.rst
+++ b/Doc/howto/functional.rst
@@ -1072,8 +1072,8 @@ write the obvious :keyword:`for` loop::
A related function is :func:`itertools.accumulate(iterable, func=operator.add)
`. It performs the same calculation, but instead of
-returning only the final result, :func:`accumulate` returns an iterator that
-also yields each partial result::
+returning only the final result, :func:`~itertools.accumulate` returns an iterator
+that also yields each partial result::
itertools.accumulate([1, 2, 3, 4, 5]) =>
1, 3, 6, 10, 15
@@ -1208,8 +1208,8 @@ General
-------
**Structure and Interpretation of Computer Programs**, by Harold Abelson and
-Gerald Jay Sussman with Julie Sussman. Full text at
-https://mitpress.mit.edu/sicp/. In this classic textbook of computer science,
+Gerald Jay Sussman with Julie Sussman. The book can be found at
+https://mitpress.mit.edu/sicp. In this classic textbook of computer science,
chapters 2 and 3 discuss the use of sequences and streams to organize the data
flow inside a program. The book uses Scheme for its examples, but many of the
design approaches described in these chapters are applicable to functional-style
diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst
index f521276a5a83c54..a835bb5f13bd1c9 100644
--- a/Doc/howto/index.rst
+++ b/Doc/howto/index.rst
@@ -28,9 +28,9 @@ Currently, the HOWTOs are:
urllib2.rst
argparse.rst
ipaddress.rst
- clinic.rst
instrumentation.rst
perf_profiling.rst
annotations.rst
isolating-extensions.rst
+ timerfd.rst
diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst
index 4ce15c69dac90be..9c99fcecce1fcbd 100644
--- a/Doc/howto/instrumentation.rst
+++ b/Doc/howto/instrumentation.rst
@@ -13,9 +13,9 @@ DTrace and SystemTap are monitoring tools, each providing a way to inspect
what the processes on a computer system are doing. They both use
domain-specific languages allowing a user to write scripts which:
- - filter which processes are to be observed
- - gather data from the processes of interest
- - generate reports on the data
+- filter which processes are to be observed
+- gather data from the processes of interest
+- generate reports on the data
As of Python 3.6, CPython can be built with embedded "markers", also
known as "probes", that can be observed by a DTrace or SystemTap script,
@@ -246,11 +246,9 @@ The output looks like this:
where the columns are:
- - time in microseconds since start of script
-
- - name of executable
-
- - PID of process
+- time in microseconds since start of script
+- name of executable
+- PID of process
and the remainder indicates the call/return hierarchy as the script executes.
@@ -292,11 +290,11 @@ Available static markers
.. object:: function__return(str filename, str funcname, int lineno)
- This marker is the converse of :c:func:`function__entry`, and indicates that
+ This marker is the converse of :c:func:`!function__entry`, and indicates that
execution of a Python function has ended (either via ``return``, or via an
exception). It is only triggered for pure-Python (bytecode) functions.
- The arguments are the same as for :c:func:`function__entry`
+ The arguments are the same as for :c:func:`!function__entry`
.. object:: line(str filename, str funcname, int lineno)
@@ -304,7 +302,7 @@ Available static markers
the equivalent of line-by-line tracing with a Python profiler. It is
not triggered within C functions.
- The arguments are the same as for :c:func:`function__entry`.
+ The arguments are the same as for :c:func:`!function__entry`.
.. object:: gc__start(int generation)
diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst
index 2eddb582da7c248..835c0afad7c6c82 100644
--- a/Doc/howto/isolating-extensions.rst
+++ b/Doc/howto/isolating-extensions.rst
@@ -1,5 +1,7 @@
.. highlight:: c
+.. _isolating-extensions-howto:
+
***************************
Isolating Extension Modules
***************************
@@ -62,7 +64,7 @@ Enter Per-Module State
Instead of focusing on per-interpreter state, Python's C API is evolving
to better support the more granular *per-module* state.
-This means that C-level data is be attached to a *module object*.
+This means that C-level data should be attached to a *module object*.
Each interpreter creates its own module object, keeping the data separate.
For testing the isolation, multiple module objects corresponding to a single
extension can even be loaded in a single interpreter.
@@ -298,10 +300,10 @@ Watch out for the following two points in particular (but note that this is not
a comprehensive list):
* Unlike static types, heap type objects are mutable by default.
- Use the :c:data:`Py_TPFLAGS_IMMUTABLETYPE` flag to prevent mutability.
+ Use the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag to prevent mutability.
* Heap types inherit :c:member:`~PyTypeObject.tp_new` by default,
so it may become possible to instantiate them from Python code.
- You can prevent this with the :c:data:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag.
+ You can prevent this with the :c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag.
Defining Heap Types
@@ -333,16 +335,48 @@ To avoid memory leaks, instances of heap types must implement the
garbage collection protocol.
That is, heap types should:
-- Have the :c:data:`Py_TPFLAGS_HAVE_GC` flag.
+- Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag.
- Define a traverse function using ``Py_tp_traverse``, which
visits the type (e.g. using :c:expr:`Py_VISIT(Py_TYPE(self))`).
-Please refer to the :ref:`the documentation ` of
-:c:data:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse`
+Please refer to the the documentation of
+:c:macro:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse`
for additional considerations.
-If your traverse function delegates to the ``tp_traverse`` of its base class
-(or another type), ensure that ``Py_TYPE(self)`` is visited only once.
+The API for defining heap types grew organically, leaving it
+somewhat awkward to use in its current state.
+The following sections will guide you through common issues.
+
+
+``tp_traverse`` in Python 3.8 and lower
+.......................................
+
+The requirement to visit the type from ``tp_traverse`` was added in Python 3.9.
+If you support Python 3.8 and lower, the traverse function must *not*
+visit the type, so it must be more complicated::
+
+ static int my_traverse(PyObject *self, visitproc visit, void *arg)
+ {
+ if (Py_Version >= 0x03090000) {
+ Py_VISIT(Py_TYPE(self));
+ }
+ return 0;
+ }
+
+Unfortunately, :c:data:`Py_Version` was only added in Python 3.11.
+As a replacement, use:
+
+* :c:macro:`PY_VERSION_HEX`, if not using the stable ABI, or
+* :py:data:`sys.version_info` (via :c:func:`PySys_GetObject` and
+ :c:func:`PyArg_ParseTuple`).
+
+
+Delegating ``tp_traverse``
+..........................
+
+If your traverse function delegates to the :c:member:`~PyTypeObject.tp_traverse`
+of its base class (or another type), ensure that ``Py_TYPE(self)`` is visited
+only once.
Note that only heap type are expected to visit the type in ``tp_traverse``.
For example, if your traverse function includes::
@@ -354,11 +388,70 @@ For example, if your traverse function includes::
if (base->tp_flags & Py_TPFLAGS_HEAPTYPE) {
// a heap type's tp_traverse already visited Py_TYPE(self)
} else {
- Py_VISIT(Py_TYPE(self));
+ if (Py_Version >= 0x03090000) {
+ Py_VISIT(Py_TYPE(self));
+ }
}
-It is not necessary to handle the type's reference count in ``tp_new``
-and ``tp_clear``.
+It is not necessary to handle the type's reference count in
+:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_clear`.
+
+
+Defining ``tp_dealloc``
+.......................
+
+If your type has a custom :c:member:`~PyTypeObject.tp_dealloc` function,
+it needs to:
+
+- call :c:func:`PyObject_GC_UnTrack` before any fields are invalidated, and
+- decrement the reference count of the type.
+
+To keep the type valid while ``tp_free`` is called, the type's refcount needs
+to be decremented *after* the instance is deallocated. For example::
+
+ static void my_dealloc(PyObject *self)
+ {
+ PyObject_GC_UnTrack(self);
+ ...
+ PyTypeObject *type = Py_TYPE(self);
+ type->tp_free(self);
+ Py_DECREF(type);
+ }
+
+The default ``tp_dealloc`` function does this, so
+if your type does *not* override
+``tp_dealloc`` you don't need to add it.
+
+
+Not overriding ``tp_free``
+..........................
+
+The :c:member:`~PyTypeObject.tp_free` slot of a heap type must be set to
+:c:func:`PyObject_GC_Del`.
+This is the default; do not override it.
+
+
+Avoiding ``PyObject_New``
+.........................
+
+GC-tracked objects need to be allocated using GC-aware functions.
+
+If you use use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`:
+
+- Get and call type's :c:member:`~PyTypeObject.tp_alloc` slot, if possible.
+ That is, replace ``TYPE *o = PyObject_New(TYPE, typeobj)`` with::
+
+ TYPE *o = typeobj->tp_alloc(typeobj, 0);
+
+ Replace ``o = PyObject_NewVar(TYPE, typeobj, size)`` with the same,
+ but use size instead of the 0.
+
+- If the above is not possible (e.g. inside a custom ``tp_alloc``),
+ call :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`::
+
+ TYPE *o = PyObject_GC_New(TYPE, typeobj);
+
+ TYPE *o = PyObject_GC_NewVar(TYPE, typeobj, size);
Module State Access from Classes
@@ -372,7 +465,7 @@ To save a some tedious error-handling boilerplate code, you can combine
these two steps with :c:func:`PyType_GetModuleState`, resulting in::
my_struct *state = (my_struct*)PyType_GetModuleState(type);
- if (state === NULL) {
+ if (state == NULL) {
return NULL;
}
@@ -411,7 +504,7 @@ that subclass, which may be defined in different module than yours.
pass
For a method to get its "defining class", it must use the
-:data:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS`
+:ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `
:c:type:`calling convention `
and the corresponding :c:type:`PyCMethod` signature::
@@ -435,7 +528,7 @@ For example::
PyObject *kwnames)
{
my_struct *state = (my_struct*)PyType_GetModuleState(defining_class);
- if (state === NULL) {
+ if (state == NULL) {
return NULL;
}
... // rest of logic
@@ -461,13 +554,13 @@ Module State Access from Slot Methods, Getters and Setters
.. After adding to limited API:
- If you use the :ref:`limited API ,
+ If you use the :ref:`limited API `,
you must update ``Py_LIMITED_API`` to ``0x030b0000``, losing ABI
compatibility with earlier versions.
Slot methods—the fast C equivalents for special methods, such as
:c:member:`~PyNumberMethods.nb_add` for :py:attr:`~object.__add__` or
-:c:member:`~PyType.tp_new` for initialization—have a very simple API that
+:c:member:`~PyTypeObject.tp_new` for initialization—have a very simple API that
doesn't allow passing in the defining class, unlike with :c:type:`PyCMethod`.
The same goes for getters and setters defined with
:c:type:`PyGetSetDef`.
@@ -479,18 +572,18 @@ to get the state::
PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &module_def);
my_struct *state = (my_struct*)PyModule_GetState(module);
- if (state === NULL) {
+ if (state == NULL) {
return NULL;
}
-``PyType_GetModuleByDef`` works by searching the
+:c:func:`!PyType_GetModuleByDef` works by searching the
:term:`method resolution order` (i.e. all superclasses) for the first
superclass that has a corresponding module.
.. note::
In very exotic cases (inheritance chains spanning multiple modules
- created from the same definition), ``PyType_GetModuleByDef`` might not
+ created from the same definition), :c:func:`!PyType_GetModuleByDef` might not
return the module of the true defining class. However, it will always
return a module with the same definition, ensuring a compatible
C memory layout.
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst
index bf6f54a841a7b9d..588f5a0a53ded0a 100644
--- a/Doc/howto/logging-cookbook.rst
+++ b/Doc/howto/logging-cookbook.rst
@@ -307,7 +307,7 @@ Suppose you configure logging with the following JSON:
"class": "logging.StreamHandler",
"level": "INFO",
"formatter": "simple",
- "stream": "ext://sys.stdout",
+ "stream": "ext://sys.stdout"
},
"stderr": {
"class": "logging.StreamHandler",
@@ -340,10 +340,12 @@ adding a ``filters`` section parallel to ``formatters`` and ``handlers``:
.. code-block:: json
- "filters": {
- "warnings_and_below": {
- "()" : "__main__.filter_maker",
- "level": "WARNING"
+ {
+ "filters": {
+ "warnings_and_below": {
+ "()" : "__main__.filter_maker",
+ "level": "WARNING"
+ }
}
}
@@ -351,12 +353,14 @@ and changing the section on the ``stdout`` handler to add it:
.. code-block:: json
- "stdout": {
- "class": "logging.StreamHandler",
- "level": "INFO",
- "formatter": "simple",
- "stream": "ext://sys.stdout",
- "filters": ["warnings_and_below"]
+ {
+ "stdout": {
+ "class": "logging.StreamHandler",
+ "level": "INFO",
+ "formatter": "simple",
+ "stream": "ext://sys.stdout",
+ "filters": ["warnings_and_below"]
+ }
}
A filter is just a function, so we can define the ``filter_maker`` (a factory
@@ -757,7 +761,7 @@ printed on the console; on the server side, you should see something like:
Note that there are some security issues with pickle in some scenarios. If
these affect you, you can use an alternative serialization scheme by overriding
-the :meth:`~handlers.SocketHandler.makePickle` method and implementing your
+the :meth:`~SocketHandler.makePickle` method and implementing your
alternative there, as well as adapting the above script to use your alternative
serialization.
@@ -831,6 +835,8 @@ To test these files, do the following in a POSIX environment:
You may need to tweak the configuration files in the unlikely event that the
configured ports clash with something else in your test environment.
+.. currentmodule:: logging
+
.. _context-info:
Adding contextual information to your logging output
@@ -1131,7 +1137,7 @@ each request is handled by a thread:
'context can be used to '
'populate logs')
aa = ap.add_argument
- aa('--count', '-c', default=100, help='How many requests to simulate')
+ aa('--count', '-c', type=int, default=100, help='How many requests to simulate')
options = ap.parse_args()
# Create the dummy webapps and put them in a list which we can use to select
@@ -1542,7 +1548,7 @@ Sometimes you want to let a log file grow to a certain size, then open a new
file and log to that. You may want to keep a certain number of these files, and
when that many files have been created, rotate the files so that the number of
files and the size of the files both remain bounded. For this usage pattern, the
-logging package provides a :class:`~handlers.RotatingFileHandler`::
+logging package provides a :class:`RotatingFileHandler`::
import glob
import logging
@@ -1590,6 +1596,8 @@ and each time it reaches the size limit it is renamed with the suffix
Obviously this example sets the log length much too small as an extreme
example. You would want to set *maxBytes* to an appropriate value.
+.. currentmodule:: logging
+
.. _format-styles:
Use of alternative formatting styles
@@ -1720,7 +1728,7 @@ when (and if) the logged message is actually about to be output to a log by a
handler. So the only slightly unusual thing which might trip you up is that the
parentheses go around the format string and the arguments, not just the format
string. That's because the __ notation is just syntax sugar for a constructor
-call to one of the XXXMessage classes.
+call to one of the :samp:`{XXX}Message` classes.
If you prefer, you can use a :class:`LoggerAdapter` to achieve a similar effect
to the above, as in the following example::
@@ -1836,6 +1844,7 @@ However, it should be borne in mind that each link in the chain adds run-time
overhead to all logging operations, and the technique should only be used when
the use of a :class:`Filter` does not provide the desired result.
+.. currentmodule:: logging.handlers
.. _zeromq-handlers:
@@ -1913,6 +1922,8 @@ of queues, for example a ZeroMQ 'subscribe' socket. Here's an example::
:ref:`A more advanced logging tutorial `
+.. currentmodule:: logging
+
An example dictionary-based configuration
-----------------------------------------
@@ -1982,26 +1993,47 @@ Using a rotator and namer to customize log rotation processing
--------------------------------------------------------------
An example of how you can define a namer and rotator is given in the following
-snippet, which shows zlib-based compression of the log file::
+runnable script, which shows gzip compression of the log file::
+
+ import gzip
+ import logging
+ import logging.handlers
+ import os
+ import shutil
def namer(name):
return name + ".gz"
def rotator(source, dest):
- with open(source, "rb") as sf:
- data = sf.read()
- compressed = zlib.compress(data, 9)
- with open(dest, "wb") as df:
- df.write(compressed)
+ with open(source, 'rb') as f_in:
+ with gzip.open(dest, 'wb') as f_out:
+ shutil.copyfileobj(f_in, f_out)
os.remove(source)
- rh = logging.handlers.RotatingFileHandler(...)
+
+ rh = logging.handlers.RotatingFileHandler('rotated.log', maxBytes=128, backupCount=5)
rh.rotator = rotator
rh.namer = namer
-These are not "true" .gz files, as they are bare compressed data, with no
-"container" such as you’d find in an actual gzip file. This snippet is just
-for illustration purposes.
+ root = logging.getLogger()
+ root.setLevel(logging.INFO)
+ root.addHandler(rh)
+ f = logging.Formatter('%(asctime)s %(message)s')
+ rh.setFormatter(f)
+ for i in range(1000):
+ root.info(f'Message no. {i + 1}')
+
+After running this, you will see six new files, five of which are compressed:
+
+.. code-block:: shell-session
+
+ $ ls rotated.log*
+ rotated.log rotated.log.2.gz rotated.log.4.gz
+ rotated.log.1.gz rotated.log.3.gz rotated.log.5.gz
+ $ zcat rotated.log.1.gz
+ 2023-01-20 02:28:17,767 Message no. 996
+ 2023-01-20 02:28:17,767 Message no. 997
+ 2023-01-20 02:28:17,767 Message no. 998
A more elaborate multiprocessing example
----------------------------------------
@@ -2517,7 +2549,7 @@ should be logged, or the ``extra`` keyword parameter to indicate additional
contextual information to be added to the log). So you cannot directly make
logging calls using :meth:`str.format` or :class:`string.Template` syntax,
because internally the logging package uses %-formatting to merge the format
-string and the variable arguments. There would no changing this while preserving
+string and the variable arguments. There would be no changing this while preserving
backward compatibility, since all logging calls which are out there in existing
code will be using %-format strings.
@@ -2612,7 +2644,7 @@ when (and if) the logged message is actually about to be output to a log by a
handler. So the only slightly unusual thing which might trip you up is that the
parentheses go around the format string and the arguments, not just the format
string. That’s because the __ notation is just syntax sugar for a constructor
-call to one of the ``XXXMessage`` classes shown above.
+call to one of the :samp:`{XXX}Message` classes shown above.
.. _filters-dictconfig:
@@ -3607,7 +3639,7 @@ refer to the comments in the code snippet for more detailed information.
Logging to syslog with RFC5424 support
--------------------------------------
-Although :rfc:`5424` dates from 2009, most syslog servers are configured by detault to
+Although :rfc:`5424` dates from 2009, most syslog servers are configured by default to
use the older :rfc:`3164`, which hails from 2001. When ``logging`` was added to Python
in 2003, it supported the earlier (and only existing) protocol at the time. Since
RFC5424 came out, as there has not been widespread deployment of it in syslog
@@ -3798,7 +3830,7 @@ then running the script results in
WARNING:demo:division by zero
As you can see, this output isn't ideal. That's because the underlying code
-which writes to ``sys.stderr`` makes mutiple writes, each of which results in a
+which writes to ``sys.stderr`` makes multiple writes, each of which results in a
separate logged line (for example, the last three lines above). To get around
this problem, you need to buffer things and only output log lines when newlines
are seen. Let's use a slghtly better implementation of ``LoggerWriter``:
@@ -3893,8 +3925,8 @@ that in other languages such as Java and C#, loggers are often static class
attributes. However, this pattern doesn't make sense in Python, where the
module (and not the class) is the unit of software decomposition.
-Adding handlers other than :class:`NullHandler` to a logger in a library
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Adding handlers other than :class:`~logging.NullHandler` to a logger in a library
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Configuring logging by adding handlers, formatters and filters is the
responsibility of the application developer, not the library developer. If you
diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst
index 145449b2dfbd9fc..7330cf675baa36d 100644
--- a/Doc/howto/logging.rst
+++ b/Doc/howto/logging.rst
@@ -418,6 +418,7 @@ The flow of log event information in loggers and handlers is illustrated in the
following diagram.
.. image:: logging_flow.png
+ :class: invert-in-dark-mode
Loggers
^^^^^^^
@@ -978,7 +979,7 @@ provided:
#. :class:`NullHandler` instances do nothing with error messages. They are used
by library developers who want to use logging, but want to avoid the 'No
- handlers could be found for logger XXX' message which can be displayed if
+ handlers could be found for logger *XXX*' message which can be displayed if
the library user has not configured logging. See :ref:`library-config` for
more information.
diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst
index ad2eb7b4d58aa57..bb1c00e0aa51d5f 100644
--- a/Doc/howto/perf_profiling.rst
+++ b/Doc/howto/perf_profiling.rst
@@ -15,21 +15,21 @@ information about the performance of your application.
that aid with the analysis of the data that it produces.
The main problem with using the ``perf`` profiler with Python applications is that
-``perf`` only allows to get information about native symbols, this is, the names of
-the functions and procedures written in C. This means that the names and file names
-of the Python functions in your code will not appear in the output of the ``perf``.
+``perf`` only gets information about native symbols, that is, the names of
+functions and procedures written in C. This means that the names and file names
+of Python functions in your code will not appear in the output of ``perf``.
Since Python 3.12, the interpreter can run in a special mode that allows Python
functions to appear in the output of the ``perf`` profiler. When this mode is
enabled, the interpreter will interpose a small piece of code compiled on the
fly before the execution of every Python function and it will teach ``perf`` the
relationship between this piece of code and the associated Python function using
-`perf map files`_.
+:doc:`perf map files <../c-api/perfmaps>`.
.. note::
- Support for the ``perf`` profiler is only currently available for Linux on
- selected architectures. Check the output of the configure build step or
+ Support for the ``perf`` profiler is currently only available for Linux on
+ select architectures. Check the output of the ``configure`` build step or
check the output of ``python -m sysconfig | grep HAVE_PERF_TRAMPOLINE``
to see if your system is supported.
@@ -52,11 +52,11 @@ For example, consider the following script:
if __name__ == "__main__":
baz(1000000)
-We can run ``perf`` to sample CPU stack traces at 9999 Hertz::
+We can run ``perf`` to sample CPU stack traces at 9999 hertz::
$ perf record -F 9999 -g -o perf.data python my_script.py
-Then we can use ``perf`` report to analyze the data:
+Then we can use ``perf report`` to analyze the data:
.. code-block:: shell-session
@@ -97,7 +97,7 @@ Then we can use ``perf`` report to analyze the data:
| | | | | |--2.97%--_PyObject_Malloc
...
-As you can see here, the Python functions are not shown in the output, only ``_Py_Eval_EvalFrameDefault`` appears
+As you can see, the Python functions are not shown in the output, only ``_PyEval_EvalFrameDefault``
(the function that evaluates the Python bytecode) shows up. Unfortunately that's not very useful because all Python
functions use the same C function to evaluate bytecode so we cannot know which Python function corresponds to which
bytecode-evaluating function.
@@ -151,7 +151,7 @@ Instead, if we run the same experiment with ``perf`` support enabled we get:
How to enable ``perf`` profiling support
----------------------------------------
-``perf`` profiling support can either be enabled from the start using
+``perf`` profiling support can be enabled either from the start using
the environment variable :envvar:`PYTHONPERFSUPPORT` or the
:option:`-X perf <-X>` option,
or dynamically using :func:`sys.activate_stack_trampoline` and
@@ -162,8 +162,7 @@ the :option:`!-X` option takes precedence over the environment variable.
Example, using the environment variable::
- $ PYTHONPERFSUPPORT=1
- $ python script.py
+ $ PYTHONPERFSUPPORT=1 python script.py
$ perf report -g -i perf.data
Example, using the :option:`!-X` option::
@@ -192,7 +191,7 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`:
How to obtain the best results
------------------------------
-For the best results, Python should be compiled with
+For best results, Python should be compiled with
``CFLAGS="-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"`` as this allows
profilers to unwind using only the frame pointer and not on DWARF debug
information. This is because as the code that is interposed to allow ``perf``
@@ -206,5 +205,3 @@ You can check if your system has been compiled with this flag by running::
If you don't see any output it means that your interpreter has not been compiled with
frame pointers and therefore it may not be able to show Python functions in the output
of ``perf``.
-
-.. _perf map files: https://github.com/torvalds/linux/blob/0513e464f9007b70b96740271a948ca5ab6e7dd7/tools/perf/Documentation/jit-interface.txt
diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst
index add1c11be534e3b..501b16d82d4d6fc 100644
--- a/Doc/howto/pyporting.rst
+++ b/Doc/howto/pyporting.rst
@@ -1,49 +1,48 @@
.. _pyporting-howto:
-*********************************
-Porting Python 2 Code to Python 3
-*********************************
+*************************************
+How to port Python 2 Code to Python 3
+*************************************
:author: Brett Cannon
.. topic:: Abstract
- With Python 3 being the future of Python while Python 2 is still in active
- use, it is good to have your project available for both major releases of
- Python. This guide is meant to help you figure out how best to support both
- Python 2 & 3 simultaneously.
+ Python 2 reached its official end-of-life at the start of 2020. This means
+ that no new bug reports, fixes, or changes will be made to Python 2 - it's
+ no longer supported.
+
+ This guide is intended to provide you with a path to Python 3 for your
+ code, that includes compatibility with Python 2 as a first step.
If you are looking to port an extension module instead of pure Python code,
please see :ref:`cporting-howto`.
- If you would like to read one core Python developer's take on why Python 3
- came into existence, you can read Nick Coghlan's `Python 3 Q & A`_ or
- Brett Cannon's `Why Python 3 exists`_.
-
+ The archived python-porting_ mailing list may contain some useful guidance.
- For help with porting, you can view the archived python-porting_ mailing list.
The Short Explanation
=====================
-To make your project be single-source Python 2/3 compatible, the basic steps
+To achieve Python 2/3 compatibility in a single code base, the basic steps
are:
#. Only worry about supporting Python 2.7
#. Make sure you have good test coverage (coverage.py_ can help;
``python -m pip install coverage``)
-#. Learn the differences between Python 2 & 3
+#. Learn the differences between Python 2 and 3
#. Use Futurize_ (or Modernize_) to update your code (e.g. ``python -m pip install future``)
#. Use Pylint_ to help make sure you don't regress on your Python 3 support
(``python -m pip install pylint``)
#. Use caniusepython3_ to find out which of your dependencies are blocking your
use of Python 3 (``python -m pip install caniusepython3``)
#. Once your dependencies are no longer blocking you, use continuous integration
- to make sure you stay compatible with Python 2 & 3 (tox_ can help test
+ to make sure you stay compatible with Python 2 and 3 (tox_ can help test
against multiple versions of Python; ``python -m pip install tox``)
-#. Consider using optional static type checking to make sure your type usage
- works in both Python 2 & 3 (e.g. use mypy_ to check your typing under both
- Python 2 & Python 3; ``python -m pip install mypy``).
+#. Consider using optional :term:`static type checking `
+ to make sure your type usage
+ works in both Python 2 and 3 (e.g. use mypy_ to check your typing under both
+ Python 2 and Python 3; ``python -m pip install mypy``).
.. note::
@@ -55,43 +54,30 @@ are:
Details
=======
-A key point about supporting Python 2 & 3 simultaneously is that you can start
-**today**! Even if your dependencies are not supporting Python 3 yet that does
-not mean you can't modernize your code **now** to support Python 3. Most changes
-required to support Python 3 lead to cleaner code using newer practices even in
-Python 2 code.
+Even if other factors - say, dependencies over which you have no control -
+still require you to support Python 2, that does not prevent you taking the
+step of including Python 3 support.
-Another key point is that modernizing your Python 2 code to also support
-Python 3 is largely automated for you. While you might have to make some API
-decisions thanks to Python 3 clarifying text data versus binary data, the
-lower-level work is now mostly done for you and thus can at least benefit from
-the automated changes immediately.
+Most changes required to support Python 3 lead to cleaner code using newer
+practices even in Python 2 code.
-Keep those key points in mind while you read on about the details of porting
-your code to support Python 2 & 3 simultaneously.
+Different versions of Python 2
+------------------------------
-Drop support for Python 2.6 and older
--------------------------------------
+Ideally, your code should be compatible with Python 2.7, which was the
+last supported version of Python 2.
-While you can make Python 2.5 work with Python 3, it is **much** easier if you
-only have to work with Python 2.7. If dropping Python 2.5 is not an
-option then the six_ project can help you support Python 2.5 & 3 simultaneously
-(``python -m pip install six``). Do realize, though, that nearly all the projects listed
-in this HOWTO will not be available to you.
+Some of the tools mentioned in this guide will not work with Python 2.6.
-If you are able to skip Python 2.5 and older, then the required changes
-to your code should continue to look and feel like idiomatic Python code. At
-worst you will have to use a function instead of a method in some instances or
-have to import a function instead of using a built-in one, but otherwise the
-overall transformation should not feel foreign to you.
+If absolutely necessary, the six_ project can help you support Python 2.5 and
+3 simultaneously. Do realize, though, that nearly all the projects listed in
+this guide will not be available to you.
-But you should aim for only supporting Python 2.7. Python 2.6 is no longer
-freely supported and thus is not receiving bugfixes. This means **you** will have
-to work around any issues you come across with Python 2.6. There are also some
-tools mentioned in this HOWTO which do not support Python 2.6 (e.g., Pylint_),
-and this will become more commonplace as time goes on. It will simply be easier
-for you if you only support the versions of Python that you have to support.
+If you are able to skip Python 2.5 and older, the required changes to your
+code will be minimal. At worst you will have to use a function instead of a
+method in some instances or have to import a function instead of using a
+built-in one.
Make sure you specify the proper version support in your ``setup.py`` file
@@ -118,62 +104,57 @@ coverage). If you don't already have a tool to measure test coverage then
coverage.py_ is recommended.
-Learn the differences between Python 2 & 3
--------------------------------------------
+Be aware of the differences between Python 2 and 3
+--------------------------------------------------
Once you have your code well-tested you are ready to begin porting your code to
Python 3! But to fully understand how your code is going to change and what
you want to look out for while you code, you will want to learn what changes
-Python 3 makes in terms of Python 2. Typically the two best ways of doing that
-is reading the :ref:`"What's New" ` doc for each release of Python 3 and the
-`Porting to Python 3`_ book (which is free online). There is also a handy
-`cheat sheet`_ from the Python-Future project.
+Python 3 makes in terms of Python 2.
+
+Some resources for understanding the differences and their implications for you
+code:
+
+* the :ref:`"What's New" ` doc for each release of Python 3
+* the `Porting to Python 3`_ book (which is free online)
+* the handy `cheat sheet`_ from the Python-Future project.
Update your code
----------------
-Once you feel like you know what is different in Python 3 compared to Python 2,
-it's time to update your code! You have a choice between two tools in porting
-your code automatically: Futurize_ and Modernize_. Which tool you choose will
-depend on how much like Python 3 you want your code to be. Futurize_ does its
-best to make Python 3 idioms and practices exist in Python 2, e.g. backporting
-the ``bytes`` type from Python 3 so that you have semantic parity between the
-major versions of Python. Modernize_,
-on the other hand, is more conservative and targets a Python 2/3 subset of
-Python, directly relying on six_ to help provide compatibility. As Python 3 is
-the future, it might be best to consider Futurize to begin adjusting to any new
-practices that Python 3 introduces which you are not accustomed to yet.
-
-Regardless of which tool you choose, they will update your code to run under
-Python 3 while staying compatible with the version of Python 2 you started with.
-Depending on how conservative you want to be, you may want to run the tool over
-your test suite first and visually inspect the diff to make sure the
-transformation is accurate. After you have transformed your test suite and
-verified that all the tests still pass as expected, then you can transform your
-application code knowing that any tests which fail is a translation failure.
+There are tools available that can port your code automatically.
+
+Futurize_ does its best to make Python 3 idioms and practices exist in Python
+2, e.g. backporting the ``bytes`` type from Python 3 so that you have
+semantic parity between the major versions of Python. This is the better
+approach for most cases.
+
+Modernize_, on the other hand, is more conservative and targets a Python 2/3
+subset of Python, directly relying on six_ to help provide compatibility.
+
+A good approach is to run the tool over your test suite first and visually
+inspect the diff to make sure the transformation is accurate. After you have
+transformed your test suite and verified that all the tests still pass as
+expected, then you can transform your application code knowing that any tests
+which fail is a translation failure.
Unfortunately the tools can't automate everything to make your code work under
-Python 3 and so there are a handful of things you will need to update manually
-to get full Python 3 support (which of these steps are necessary vary between
-the tools). Read the documentation for the tool you choose to use to see what it
-fixes by default and what it can do optionally to know what will (not) be fixed
-for you and what you may have to fix on your own (e.g. using ``io.open()`` over
-the built-in ``open()`` function is off by default in Modernize). Luckily,
-though, there are only a couple of things to watch out for which can be
-considered large issues that may be hard to debug if not watched for.
+Python 3, and you will also need to read the tools' documentation in case some
+options you need are turned off by default.
+Key issues to be aware of and check for:
Division
++++++++
-In Python 3, ``5 / 2 == 2.5`` and not ``2``; all division between ``int`` values
-result in a ``float``. This change has actually been planned since Python 2.2
-which was released in 2002. Since then users have been encouraged to add
-``from __future__ import division`` to any and all files which use the ``/`` and
-``//`` operators or to be running the interpreter with the ``-Q`` flag. If you
-have not been doing this then you will need to go through your code and do two
-things:
+In Python 3, ``5 / 2 == 2.5`` and not ``2`` as it was in Python 2; all
+division between ``int`` values result in a ``float``. This change has
+actually been planned since Python 2.2 which was released in 2002. Since then
+users have been encouraged to add ``from __future__ import division`` to any
+and all files which use the ``/`` and ``//`` operators or to be running the
+interpreter with the ``-Q`` flag. If you have not been doing this then you
+will need to go through your code and do two things:
#. Add ``from __future__ import division`` to your files
#. Update any division operator as necessary to either use ``//`` to use floor
@@ -197,30 +178,29 @@ specific type. This complicated the situation especially for anyone supporting
multiple languages as APIs wouldn't bother explicitly supporting ``unicode``
when they claimed text data support.
-To make the distinction between text and binary data clearer and more
-pronounced, Python 3 did what most languages created in the age of the internet
-have done and made text and binary data distinct types that cannot blindly be
-mixed together (Python predates widespread access to the internet). For any code
-that deals only with text or only binary data, this separation doesn't pose an
-issue. But for code that has to deal with both, it does mean you might have to
-now care about when you are using text compared to binary data, which is why
-this cannot be entirely automated.
-
-To start, you will need to decide which APIs take text and which take binary
-(it is **highly** recommended you don't design APIs that can take both due to
-the difficulty of keeping the code working; as stated earlier it is difficult to
-do well). In Python 2 this means making sure the APIs that take text can work
-with ``unicode`` and those that work with binary data work with the
-``bytes`` type from Python 3 (which is a subset of ``str`` in Python 2 and acts
-as an alias for ``bytes`` type in Python 2). Usually the biggest issue is
-realizing which methods exist on which types in Python 2 & 3 simultaneously
-(for text that's ``unicode`` in Python 2 and ``str`` in Python 3, for binary
-that's ``str``/``bytes`` in Python 2 and ``bytes`` in Python 3). The following
-table lists the **unique** methods of each data type across Python 2 & 3
-(e.g., the ``decode()`` method is usable on the equivalent binary data type in
-either Python 2 or 3, but it can't be used by the textual data type consistently
-between Python 2 and 3 because ``str`` in Python 3 doesn't have the method). Do
-note that as of Python 3.5 the ``__mod__`` method was added to the bytes type.
+Python 3 made text and binary data distinct types that cannot simply be mixed
+together. For any code that deals only with text or only binary data, this
+separation doesn't pose an issue. But for code that has to deal with both, it
+does mean you might have to now care about when you are using text compared
+to binary data, which is why this cannot be entirely automated.
+
+Decide which APIs take text and which take binary (it is **highly** recommended
+you don't design APIs that can take both due to the difficulty of keeping the
+code working; as stated earlier it is difficult to do well). In Python 2 this
+means making sure the APIs that take text can work with ``unicode`` and those
+that work with binary data work with the ``bytes`` type from Python 3
+(which is a subset of ``str`` in Python 2 and acts as an alias for ``bytes``
+type in Python 2). Usually the biggest issue is realizing which methods exist
+on which types in Python 2 and 3 simultaneously (for text that's ``unicode``
+in Python 2 and ``str`` in Python 3, for binary that's ``str``/``bytes`` in
+Python 2 and ``bytes`` in Python 3).
+
+The following table lists the **unique** methods of each data type across
+Python 2 and 3 (e.g., the ``decode()`` method is usable on the equivalent binary
+data type in either Python 2 or 3, but it can't be used by the textual data
+type consistently between Python 2 and 3 because ``str`` in Python 3 doesn't
+have the method). Do note that as of Python 3.5 the ``__mod__`` method was
+added to the bytes type.
======================== =====================
**Text data** **Binary data**
@@ -246,12 +226,11 @@ having to keep track of what type of data you are working with.
The next issue is making sure you know whether the string literals in your code
represent text or binary data. You should add a ``b`` prefix to any
literal that presents binary data. For text you should add a ``u`` prefix to
-the text literal. (there is a :mod:`__future__` import to force all unspecified
+the text literal. (There is a :mod:`__future__` import to force all unspecified
literals to be Unicode, but usage has shown it isn't as effective as adding a
``b`` or ``u`` prefix to all literals explicitly)
-As part of this dichotomy you also need to be careful about opening files.
-Unless you have been working on Windows, there is a chance you have not always
+You also need to be careful about opening files. Possibly you have not always
bothered to add the ``b`` mode when opening a binary file (e.g., ``rb`` for
binary reading). Under Python 3, binary files and text files are clearly
distinct and mutually incompatible; see the :mod:`io` module for details.
@@ -265,7 +244,7 @@ outdated practice of using :func:`codecs.open` as that's only necessary for
keeping compatibility with Python 2.5.
The constructors of both ``str`` and ``bytes`` have different semantics for the
-same arguments between Python 2 & 3. Passing an integer to ``bytes`` in Python 2
+same arguments between Python 2 and 3. Passing an integer to ``bytes`` in Python 2
will give you the string representation of the integer: ``bytes(3) == '3'``.
But in Python 3, an integer argument to ``bytes`` will give you a bytes object
as long as the integer specified, filled with null bytes:
@@ -400,7 +379,7 @@ Use continuous integration to stay compatible
---------------------------------------------
Once you are able to fully run under Python 3 you will want to make sure your
-code always works under both Python 2 & 3. Probably the best tool for running
+code always works under both Python 2 and 3. Probably the best tool for running
your tests under multiple Python interpreters is tox_. You can then integrate
tox with your continuous integration system so that you never accidentally break
Python 2 or 3 support.
@@ -413,16 +392,11 @@ separation of text/binary data handling or indexing on bytes you wouldn't easily
find the mistake. This flag will raise an exception when these kinds of
comparisons occur, making the mistake much easier to track down.
-And that's mostly it! At this point your code base is compatible with both
-Python 2 and 3 simultaneously. Your testing will also be set up so that you
-don't accidentally break Python 2 or 3 compatibility regardless of which version
-you typically run your tests under while developing.
-
Consider using optional static type checking
--------------------------------------------
-Another way to help port your code is to use a static type checker like
+Another way to help port your code is to use a :term:`static type checker` like
mypy_ or pytype_ on your code. These tools can be used to analyze your code as
if it's being run under Python 2, then you can run the tool a second time as if
your code is running under Python 3. By running a static type checker twice like
@@ -438,7 +412,7 @@ to make sure everything functions as expected in both versions of Python.
.. _Futurize: https://python-future.org/automatic_conversion.html
.. _importlib2: https://pypi.org/project/importlib2
.. _Modernize: https://python-modernize.readthedocs.io/
-.. _mypy: http://mypy-lang.org/
+.. _mypy: https://mypy-lang.org/
.. _Porting to Python 3: http://python3porting.com/
.. _Pylint: https://pypi.org/project/pylint
diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst
index 655df59e27b641e..5e2f9a9d1837fe0 100644
--- a/Doc/howto/regex.rst
+++ b/Doc/howto/regex.rst
@@ -245,6 +245,9 @@ You can omit either *m* or *n*; in that case, a reasonable value is assumed for
the missing value. Omitting *m* is interpreted as a lower limit of 0, while
omitting *n* results in an upper bound of infinity.
+The simplest case ``{m}`` matches the preceding item exactly *m* times.
+For example, ``a/{2}b`` will only match ``'a//b'``.
+
Readers of a reductionist bent may notice that the three other quantifiers can
all be expressed using this notation. ``{0,}`` is the same as ``*``, ``{1,}``
is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use
@@ -518,6 +521,8 @@ cache.
Compilation Flags
-----------------
+.. currentmodule:: re
+
Compilation flags let you modify some aspects of how regular expressions work.
Flags are available in the :mod:`re` module under two names, a long name such as
:const:`IGNORECASE` and a short, one-letter form such as :const:`I`. (If you're
diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst
index decce12bf3faf6b..38dd09f0a721d2f 100644
--- a/Doc/howto/sorting.rst
+++ b/Doc/howto/sorting.rst
@@ -273,7 +273,7 @@ Odds and Ends
* The sort routines use ``<`` when making comparisons
between two objects. So, it is easy to add a standard sort order to a class by
- defining an :meth:`__lt__` method:
+ defining an :meth:`~object.__lt__` method:
.. doctest::
@@ -281,8 +281,8 @@ Odds and Ends
>>> sorted(student_objects)
[('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)]
- However, note that ``<`` can fall back to using :meth:`__gt__` if
- :meth:`__lt__` is not implemented (see :func:`object.__lt__`).
+ However, note that ``<`` can fall back to using :meth:`~object.__gt__` if
+ :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__`).
* Key functions need not depend directly on the objects being sorted. A key
function can also access external resources. For instance, if the student grades
diff --git a/Doc/howto/timerfd.rst b/Doc/howto/timerfd.rst
new file mode 100644
index 000000000000000..98f0294f9d082d3
--- /dev/null
+++ b/Doc/howto/timerfd.rst
@@ -0,0 +1,230 @@
+.. _timerfd-howto:
+
+*****************************
+ timer file descriptor HOWTO
+*****************************
+
+:Release: 1.13
+
+This HOWTO discusses Python's support for the linux timer file descriptor.
+
+
+Examples
+========
+
+The following example shows how to use a timer file descriptor
+to execute a function twice a second:
+
+.. code-block:: python
+
+ # Practical scripts should use really use a non-blocking timer,
+ # we use a blocking timer here for simplicity.
+ import os, time
+
+ # Create the timer file descriptor
+ fd = os.timerfd_create(time.CLOCK_REALTIME)
+
+ # Start the timer in 1 second, with an interval of half a second
+ os.timerfd_settime(fd, initial=1, interval=0.5)
+
+ try:
+ # Process timer events four times.
+ for _ in range(4):
+ # read() will block until the timer expires
+ _ = os.read(fd, 8)
+ print("Timer expired")
+ finally:
+ # Remember to close the timer file descriptor!
+ os.close(fd)
+
+To avoid the precision loss caused by the :class:`float` type,
+timer file descriptors allow specifying initial expiration and interval
+in integer nanoseconds with ``_ns`` variants of the functions.
+
+This example shows how :func:`~select.epoll` can be used with timer file
+descriptors to wait until the file descriptor is ready for reading:
+
+.. code-block:: python
+
+ import os, time, select, socket, sys
+
+ # Create an epoll object
+ ep = select.epoll()
+
+ # In this example, use loopback address to send "stop" command to the server.
+ #
+ # $ telnet 127.0.0.1 1234
+ # Trying 127.0.0.1...
+ # Connected to 127.0.0.1.
+ # Escape character is '^]'.
+ # stop
+ # Connection closed by foreign host.
+ #
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.bind(("127.0.0.1", 1234))
+ sock.setblocking(False)
+ sock.listen(1)
+ ep.register(sock, select.EPOLLIN)
+
+ # Create timer file descriptors in non-blocking mode.
+ num = 3
+ fds = []
+ for _ in range(num):
+ fd = os.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK)
+ fds.append(fd)
+ # Register the timer file descriptor for read events
+ ep.register(fd, select.EPOLLIN)
+
+ # Start the timer with os.timerfd_settime_ns() in nanoseconds.
+ # Timer 1 fires every 0.25 seconds; timer 2 every 0.5 seconds; etc
+ for i, fd in enumerate(fds, start=1):
+ one_sec_in_nsec = 10**9
+ i = i * one_sec_in_nsec
+ os.timerfd_settime_ns(fd, initial=i//4, interval=i//4)
+
+ timeout = 3
+ try:
+ conn = None
+ is_active = True
+ while is_active:
+ # Wait for the timer to expire for 3 seconds.
+ # epoll.poll() returns a list of (fd, event) pairs.
+ # fd is a file descriptor.
+ # sock and conn[=returned value of socket.accept()] are socket objects, not file descriptors.
+ # So use sock.fileno() and conn.fileno() to get the file descriptors.
+ events = ep.poll(timeout)
+
+ # If more than one timer file descriptors are ready for reading at once,
+ # epoll.poll() returns a list of (fd, event) pairs.
+ #
+ # In this example settings,
+ # 1st timer fires every 0.25 seconds in 0.25 seconds. (0.25, 0.5, 0.75, 1.0, ...)
+ # 2nd timer every 0.5 seconds in 0.5 seconds. (0.5, 1.0, 1.5, 2.0, ...)
+ # 3rd timer every 0.75 seconds in 0.75 seconds. (0.75, 1.5, 2.25, 3.0, ...)
+ #
+ # In 0.25 seconds, only 1st timer fires.
+ # In 0.5 seconds, 1st timer and 2nd timer fires at once.
+ # In 0.75 seconds, 1st timer and 3rd timer fires at once.
+ # In 1.5 seconds, 1st timer, 2nd timer and 3rd timer fires at once.
+ #
+ # If a timer file descriptor is signaled more than once since
+ # the last os.read() call, os.read() returns the nubmer of signaled
+ # as host order of class bytes.
+ print(f"Signaled events={events}")
+ for fd, event in events:
+ if event & select.EPOLLIN:
+ if fd == sock.fileno():
+ # Check if there is a connection request.
+ print(f"Accepting connection {fd}")
+ conn, addr = sock.accept()
+ conn.setblocking(False)
+ print(f"Accepted connection {conn} from {addr}")
+ ep.register(conn, select.EPOLLIN)
+ elif conn and fd == conn.fileno():
+ # Check if there is data to read.
+ print(f"Reading data {fd}")
+ data = conn.recv(1024)
+ if data:
+ # You should catch UnicodeDecodeError exception for safety.
+ cmd = data.decode()
+ if cmd.startswith("stop"):
+ print(f"Stopping server")
+ is_active = False
+ else:
+ print(f"Unknown command: {cmd}")
+ else:
+ # No more data, close connection
+ print(f"Closing connection {fd}")
+ ep.unregister(conn)
+ conn.close()
+ conn = None
+ elif fd in fds:
+ print(f"Reading timer {fd}")
+ count = int.from_bytes(os.read(fd, 8), byteorder=sys.byteorder)
+ print(f"Timer {fds.index(fd) + 1} expired {count} times")
+ else:
+ print(f"Unknown file descriptor {fd}")
+ finally:
+ for fd in fds:
+ ep.unregister(fd)
+ os.close(fd)
+ ep.close()
+
+This example shows how :func:`~select.select` can be used with timer file
+descriptors to wait until the file descriptor is ready for reading:
+
+.. code-block:: python
+
+ import os, time, select, socket, sys
+
+ # In this example, use loopback address to send "stop" command to the server.
+ #
+ # $ telnet 127.0.0.1 1234
+ # Trying 127.0.0.1...
+ # Connected to 127.0.0.1.
+ # Escape character is '^]'.
+ # stop
+ # Connection closed by foreign host.
+ #
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.bind(("127.0.0.1", 1234))
+ sock.setblocking(False)
+ sock.listen(1)
+
+ # Create timer file descriptors in non-blocking mode.
+ num = 3
+ fds = [os.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK)
+ for _ in range(num)]
+ select_fds = fds + [sock]
+
+ # Start the timers with os.timerfd_settime() in seconds.
+ # Timer 1 fires every 0.25 seconds; timer 2 every 0.5 seconds; etc
+ for i, fd in enumerate(fds, start=1):
+ os.timerfd_settime(fd, initial=i/4, interval=i/4)
+
+ timeout = 3
+ try:
+ conn = None
+ is_active = True
+ while is_active:
+ # Wait for the timer to expire for 3 seconds.
+ # select.select() returns a list of file descriptors or objects.
+ rfd, wfd, xfd = select.select(select_fds, select_fds, select_fds, timeout)
+ for fd in rfd:
+ if fd == sock:
+ # Check if there is a connection request.
+ print(f"Accepting connection {fd}")
+ conn, addr = sock.accept()
+ conn.setblocking(False)
+ print(f"Accepted connection {conn} from {addr}")
+ select_fds.append(conn)
+ elif conn and fd == conn:
+ # Check if there is data to read.
+ print(f"Reading data {fd}")
+ data = conn.recv(1024)
+ if data:
+ # You should catch UnicodeDecodeError exception for safety.
+ cmd = data.decode()
+ if cmd.startswith("stop"):
+ print(f"Stopping server")
+ is_active = False
+ else:
+ print(f"Unknown command: {cmd}")
+ else:
+ # No more data, close connection
+ print(f"Closing connection {fd}")
+ select_fds.remove(conn)
+ conn.close()
+ conn = None
+ elif fd in fds:
+ print(f"Reading timer {fd}")
+ count = int.from_bytes(os.read(fd, 8), byteorder=sys.byteorder)
+ print(f"Timer {fds.index(fd) + 1} expired {count} times")
+ else:
+ print(f"Unknown file descriptor {fd}")
+ finally:
+ for fd in fds:
+ os.close(fd)
+ sock.close()
+ sock = None
+
diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index ca09aee72bf879d..254fe7293553535 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -424,8 +424,8 @@ lowercase letters 'ss'.
A second tool is the :mod:`unicodedata` module's
:func:`~unicodedata.normalize` function that converts strings to one
-of several normal forms, where letters followed by a combining
-character are replaced with single characters. :func:`normalize` can
+of several normal forms, where letters followed by a combining character are
+replaced with single characters. :func:`~unicodedata.normalize` can
be used to perform string comparisons that won't falsely report
inequality if two strings use combining characters differently:
@@ -449,7 +449,7 @@ When run, this outputs:
.. code-block:: shell-session
- $ python3 compare-strs.py
+ $ python compare-strs.py
length of first string= 1
length of second string= 2
True
@@ -474,8 +474,8 @@ The Unicode Standard also specifies how to do caseless comparisons::
print(compare_caseless(single_char, multiple_chars))
-This will print ``True``. (Why is :func:`NFD` invoked twice? Because
-there are a few characters that make :meth:`casefold` return a
+This will print ``True``. (Why is :func:`!NFD` invoked twice? Because
+there are a few characters that make :meth:`~str.casefold` return a
non-normalized string, so the result needs to be normalized again. See
section 3.13 of the Unicode Standard for a discussion and an example.)
diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst
index 69af3c3a85c5d6c..570435d48866d3b 100644
--- a/Doc/howto/urllib2.rst
+++ b/Doc/howto/urllib2.rst
@@ -6,13 +6,6 @@
:Author: `Michael Foord `_
-.. note::
-
- There is a French translation of an earlier revision of this
- HOWTO, available at `urllib2 - Le Manuel manquant
- `_.
-
-
Introduction
============
@@ -86,7 +79,7 @@ response::
import urllib.request
- req = urllib.request.Request('http://www.voidspace.org.uk')
+ req = urllib.request.Request('http://python.org/')
with urllib.request.urlopen(req) as response:
the_page = response.read()
@@ -201,11 +194,11 @@ which comes after we have a look at what happens when things go wrong.
Handling Exceptions
===================
-*urlopen* raises :exc:`URLError` when it cannot handle a response (though as
+*urlopen* raises :exc:`~urllib.error.URLError` when it cannot handle a response (though as
usual with Python APIs, built-in exceptions such as :exc:`ValueError`,
:exc:`TypeError` etc. may also be raised).
-:exc:`HTTPError` is the subclass of :exc:`URLError` raised in the specific case of
+:exc:`~urllib.error.HTTPError` is the subclass of :exc:`~urllib.error.URLError` raised in the specific case of
HTTP URLs.
The exception classes are exported from the :mod:`urllib.error` module.
@@ -236,12 +229,12 @@ the status code indicates that the server is unable to fulfil the request. The
default handlers will handle some of these responses for you (for example, if
the response is a "redirection" that requests the client fetch the document from
a different URL, urllib will handle that for you). For those it can't handle,
-urlopen will raise an :exc:`HTTPError`. Typical errors include '404' (page not
+urlopen will raise an :exc:`~urllib.error.HTTPError`. Typical errors include '404' (page not
found), '403' (request forbidden), and '401' (authentication required).
See section 10 of :rfc:`2616` for a reference on all the HTTP error codes.
-The :exc:`HTTPError` instance raised will have an integer 'code' attribute, which
+The :exc:`~urllib.error.HTTPError` instance raised will have an integer 'code' attribute, which
corresponds to the error sent by the server.
Error Codes
@@ -324,7 +317,7 @@ dictionary is reproduced here for convenience ::
}
When an error is raised the server responds by returning an HTTP error code
-*and* an error page. You can use the :exc:`HTTPError` instance as a response on the
+*and* an error page. You can use the :exc:`~urllib.error.HTTPError` instance as a response on the
page returned. This means that as well as the code attribute, it also has read,
geturl, and info, methods as returned by the ``urllib.response`` module::
@@ -345,7 +338,7 @@ geturl, and info, methods as returned by the ``urllib.response`` module::
Wrapping it Up
--------------
-So if you want to be prepared for :exc:`HTTPError` *or* :exc:`URLError` there are two
+So if you want to be prepared for :exc:`~urllib.error.HTTPError` *or* :exc:`~urllib.error.URLError` there are two
basic approaches. I prefer the second approach.
Number 1
@@ -372,7 +365,7 @@ Number 1
.. note::
The ``except HTTPError`` *must* come first, otherwise ``except URLError``
- will *also* catch an :exc:`HTTPError`.
+ will *also* catch an :exc:`~urllib.error.HTTPError`.
Number 2
~~~~~~~~
@@ -398,7 +391,7 @@ Number 2
info and geturl
===============
-The response returned by urlopen (or the :exc:`HTTPError` instance) has two
+The response returned by urlopen (or the :exc:`~urllib.error.HTTPError` instance) has two
useful methods :meth:`info` and :meth:`geturl` and is defined in the module
:mod:`urllib.response`..
@@ -458,7 +451,7 @@ To illustrate creating and installing a handler we will use the
``HTTPBasicAuthHandler``. For a more detailed discussion of this subject --
including an explanation of how Basic Authentication works - see the `Basic
Authentication Tutorial
-`_.
+`__.
When authentication is required, the server sends a header (as well as the 401
error code) requesting authentication. This specifies the authentication scheme
diff --git a/Doc/includes/email-alternative.py b/Doc/includes/email-alternative.py
index df7ca6f3faa332c..26b302b495c7acc 100644
--- a/Doc/includes/email-alternative.py
+++ b/Doc/includes/email-alternative.py
@@ -8,14 +8,14 @@
# Create the base text message.
msg = EmailMessage()
-msg['Subject'] = "Ayons asperges pour le déjeuner"
+msg['Subject'] = "Pourquoi pas des asperges pour ce midi ?"
msg['From'] = Address("Pepé Le Pew", "pepe", "example.com")
msg['To'] = (Address("Penelope Pussycat", "penelope", "example.com"),
Address("Fabrette Pussycat", "fabrette", "example.com"))
msg.set_content("""\
Salut!
-Cela ressemble à un excellent recipie[1] déjeuner.
+Cette recette [1] sera sûrement un très bon repas.
[1] http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718
@@ -31,10 +31,10 @@
Salut!
- Cela ressemble à un excellent
+
Cette
- recipie
- déjeuner.
+ recette
+
sera sûrement un très bon repas.
diff --git a/Doc/includes/custom.c b/Doc/includes/newtypes/custom.c
similarity index 80%
rename from Doc/includes/custom.c
rename to Doc/includes/newtypes/custom.c
index 26ca754964733d0..5253f879360210e 100644
--- a/Doc/includes/custom.c
+++ b/Doc/includes/newtypes/custom.c
@@ -7,7 +7,7 @@ typedef struct {
} CustomObject;
static PyTypeObject CustomType = {
- PyVarObject_HEAD_INIT(NULL, 0)
+ .ob_base = PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom.Custom",
.tp_doc = PyDoc_STR("Custom objects"),
.tp_basicsize = sizeof(CustomObject),
@@ -17,7 +17,7 @@ static PyTypeObject CustomType = {
};
static PyModuleDef custommodule = {
- PyModuleDef_HEAD_INIT,
+ .m_base = PyModuleDef_HEAD_INIT,
.m_name = "custom",
.m_doc = "Example module that creates an extension type.",
.m_size = -1,
@@ -34,9 +34,7 @@ PyInit_custom(void)
if (m == NULL)
return NULL;
- Py_INCREF(&CustomType);
- if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
- Py_DECREF(&CustomType);
+ if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
Py_DECREF(m);
return NULL;
}
diff --git a/Doc/includes/custom2.c b/Doc/includes/newtypes/custom2.c
similarity index 81%
rename from Doc/includes/custom2.c
rename to Doc/includes/newtypes/custom2.c
index 2a3c59f8f04c3d9..a0222b1795209b7 100644
--- a/Doc/includes/custom2.c
+++ b/Doc/includes/newtypes/custom2.c
@@ -1,6 +1,6 @@
#define PY_SSIZE_T_CLEAN
#include
-#include "structmember.h"
+#include /* for offsetof() */
typedef struct {
PyObject_HEAD
@@ -42,7 +42,7 @@ static int
Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"first", "last", "number", NULL};
- PyObject *first = NULL, *last = NULL, *tmp;
+ PyObject *first = NULL, *last = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
&first, &last,
@@ -50,26 +50,20 @@ Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
return -1;
if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_XDECREF(tmp);
+ Py_XSETREF(self->first, Py_NewRef(first));
}
if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_XDECREF(tmp);
+ Py_XSETREF(self->last, Py_NewRef(last));
}
return 0;
}
static PyMemberDef Custom_members[] = {
- {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0,
+ {"first", Py_T_OBJECT_EX, offsetof(CustomObject, first), 0,
"first name"},
- {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0,
+ {"last", Py_T_OBJECT_EX, offsetof(CustomObject, last), 0,
"last name"},
- {"number", T_INT, offsetof(CustomObject, number), 0,
+ {"number", Py_T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
@@ -96,7 +90,7 @@ static PyMethodDef Custom_methods[] = {
};
static PyTypeObject CustomType = {
- PyVarObject_HEAD_INIT(NULL, 0)
+ .ob_base = PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom2.Custom",
.tp_doc = PyDoc_STR("Custom objects"),
.tp_basicsize = sizeof(CustomObject),
@@ -110,7 +104,7 @@ static PyTypeObject CustomType = {
};
static PyModuleDef custommodule = {
- PyModuleDef_HEAD_INIT,
+ .m_base =PyModuleDef_HEAD_INIT,
.m_name = "custom2",
.m_doc = "Example module that creates an extension type.",
.m_size = -1,
@@ -127,9 +121,7 @@ PyInit_custom2(void)
if (m == NULL)
return NULL;
- Py_INCREF(&CustomType);
- if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
- Py_DECREF(&CustomType);
+ if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
Py_DECREF(m);
return NULL;
}
diff --git a/Doc/includes/custom3.c b/Doc/includes/newtypes/custom3.c
similarity index 81%
rename from Doc/includes/custom3.c
rename to Doc/includes/newtypes/custom3.c
index 5a47530f0a6b0dc..4aeebe0a7507d10 100644
--- a/Doc/includes/custom3.c
+++ b/Doc/includes/newtypes/custom3.c
@@ -1,6 +1,6 @@
#define PY_SSIZE_T_CLEAN
#include
-#include "structmember.h"
+#include /* for offsetof() */
typedef struct {
PyObject_HEAD
@@ -42,7 +42,7 @@ static int
Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"first", "last", "number", NULL};
- PyObject *first = NULL, *last = NULL, *tmp;
+ PyObject *first = NULL, *last = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
&first, &last,
@@ -50,22 +50,16 @@ Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
return -1;
if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_DECREF(tmp);
+ Py_SETREF(self->first, Py_NewRef(first));
}
if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_DECREF(tmp);
+ Py_SETREF(self->last, Py_NewRef(last));
}
return 0;
}
static PyMemberDef Custom_members[] = {
- {"number", T_INT, offsetof(CustomObject, number), 0,
+ {"number", Py_T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
@@ -73,14 +67,12 @@ static PyMemberDef Custom_members[] = {
static PyObject *
Custom_getfirst(CustomObject *self, void *closure)
{
- Py_INCREF(self->first);
- return self->first;
+ return Py_NewRef(self->first);
}
static int
Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
{
- PyObject *tmp;
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
return -1;
@@ -90,24 +82,19 @@ Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
"The first attribute value must be a string");
return -1;
}
- tmp = self->first;
- Py_INCREF(value);
- self->first = value;
- Py_DECREF(tmp);
+ Py_SETREF(self->first, Py_NewRef(value));
return 0;
}
static PyObject *
Custom_getlast(CustomObject *self, void *closure)
{
- Py_INCREF(self->last);
- return self->last;
+ return Py_NewRef(self->last);
}
static int
Custom_setlast(CustomObject *self, PyObject *value, void *closure)
{
- PyObject *tmp;
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
return -1;
@@ -117,10 +104,7 @@ Custom_setlast(CustomObject *self, PyObject *value, void *closure)
"The last attribute value must be a string");
return -1;
}
- tmp = self->last;
- Py_INCREF(value);
- self->last = value;
- Py_DECREF(tmp);
+ Py_SETREF(self->last, Py_NewRef(value));
return 0;
}
@@ -146,7 +130,7 @@ static PyMethodDef Custom_methods[] = {
};
static PyTypeObject CustomType = {
- PyVarObject_HEAD_INIT(NULL, 0)
+ .ob_base = PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom3.Custom",
.tp_doc = PyDoc_STR("Custom objects"),
.tp_basicsize = sizeof(CustomObject),
@@ -161,7 +145,7 @@ static PyTypeObject CustomType = {
};
static PyModuleDef custommodule = {
- PyModuleDef_HEAD_INIT,
+ .m_base = PyModuleDef_HEAD_INIT,
.m_name = "custom3",
.m_doc = "Example module that creates an extension type.",
.m_size = -1,
@@ -178,9 +162,7 @@ PyInit_custom3(void)
if (m == NULL)
return NULL;
- Py_INCREF(&CustomType);
- if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
- Py_DECREF(&CustomType);
+ if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
Py_DECREF(m);
return NULL;
}
diff --git a/Doc/includes/custom4.c b/Doc/includes/newtypes/custom4.c
similarity index 84%
rename from Doc/includes/custom4.c
rename to Doc/includes/newtypes/custom4.c
index c7ee55578488ed4..3998918f68301e9 100644
--- a/Doc/includes/custom4.c
+++ b/Doc/includes/newtypes/custom4.c
@@ -1,6 +1,6 @@
#define PY_SSIZE_T_CLEAN
#include
-#include "structmember.h"
+#include /* for offsetof() */
typedef struct {
PyObject_HEAD
@@ -58,7 +58,7 @@ static int
Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"first", "last", "number", NULL};
- PyObject *first = NULL, *last = NULL, *tmp;
+ PyObject *first = NULL, *last = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
&first, &last,
@@ -66,22 +66,16 @@ Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
return -1;
if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_DECREF(tmp);
+ Py_SETREF(self->first, Py_NewRef(first));
}
if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_DECREF(tmp);
+ Py_SETREF(self->last, Py_NewRef(last));
}
return 0;
}
static PyMemberDef Custom_members[] = {
- {"number", T_INT, offsetof(CustomObject, number), 0,
+ {"number", Py_T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
@@ -89,8 +83,7 @@ static PyMemberDef Custom_members[] = {
static PyObject *
Custom_getfirst(CustomObject *self, void *closure)
{
- Py_INCREF(self->first);
- return self->first;
+ return Py_NewRef(self->first);
}
static int
@@ -105,17 +98,14 @@ Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
"The first attribute value must be a string");
return -1;
}
- Py_INCREF(value);
- Py_CLEAR(self->first);
- self->first = value;
+ Py_XSETREF(self->first, Py_NewRef(value));
return 0;
}
static PyObject *
Custom_getlast(CustomObject *self, void *closure)
{
- Py_INCREF(self->last);
- return self->last;
+ return Py_NewRef(self->last);
}
static int
@@ -130,9 +120,7 @@ Custom_setlast(CustomObject *self, PyObject *value, void *closure)
"The last attribute value must be a string");
return -1;
}
- Py_INCREF(value);
- Py_CLEAR(self->last);
- self->last = value;
+ Py_XSETREF(self->last, Py_NewRef(value));
return 0;
}
@@ -158,7 +146,7 @@ static PyMethodDef Custom_methods[] = {
};
static PyTypeObject CustomType = {
- PyVarObject_HEAD_INIT(NULL, 0)
+ .ob_base = PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom4.Custom",
.tp_doc = PyDoc_STR("Custom objects"),
.tp_basicsize = sizeof(CustomObject),
@@ -175,7 +163,7 @@ static PyTypeObject CustomType = {
};
static PyModuleDef custommodule = {
- PyModuleDef_HEAD_INIT,
+ .m_base = PyModuleDef_HEAD_INIT,
.m_name = "custom4",
.m_doc = "Example module that creates an extension type.",
.m_size = -1,
@@ -192,9 +180,7 @@ PyInit_custom4(void)
if (m == NULL)
return NULL;
- Py_INCREF(&CustomType);
- if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) {
- Py_DECREF(&CustomType);
+ if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
Py_DECREF(m);
return NULL;
}
diff --git a/Doc/includes/newtypes/pyproject.toml b/Doc/includes/newtypes/pyproject.toml
new file mode 100644
index 000000000000000..ea7937a3171473a
--- /dev/null
+++ b/Doc/includes/newtypes/pyproject.toml
@@ -0,0 +1,7 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "custom"
+version = "1"
diff --git a/Doc/includes/newtypes/setup.py b/Doc/includes/newtypes/setup.py
new file mode 100644
index 000000000000000..67f83673bcc657d
--- /dev/null
+++ b/Doc/includes/newtypes/setup.py
@@ -0,0 +1,8 @@
+from setuptools import Extension, setup
+setup(ext_modules=[
+ Extension("custom", ["custom.c"]),
+ Extension("custom2", ["custom2.c"]),
+ Extension("custom3", ["custom3.c"]),
+ Extension("custom4", ["custom4.c"]),
+ Extension("sublist", ["sublist.c"]),
+])
diff --git a/Doc/includes/sublist.c b/Doc/includes/newtypes/sublist.c
similarity index 91%
rename from Doc/includes/sublist.c
rename to Doc/includes/newtypes/sublist.c
index b36dadf07eae877..d8aba463f30ba2a 100644
--- a/Doc/includes/sublist.c
+++ b/Doc/includes/newtypes/sublist.c
@@ -58,9 +58,7 @@ PyInit_sublist(void)
if (m == NULL)
return NULL;
- Py_INCREF(&SubListType);
- if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) {
- Py_DECREF(&SubListType);
+ if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) {
Py_DECREF(m);
return NULL;
}
diff --git a/Doc/includes/test.py b/Doc/includes/newtypes/test.py
similarity index 94%
rename from Doc/includes/test.py
rename to Doc/includes/newtypes/test.py
index 09ebe3fec0bdbe3..55a5cf9f68b94ac 100644
--- a/Doc/includes/test.py
+++ b/Doc/includes/newtypes/test.py
@@ -187,13 +187,6 @@
>>> gc.enable()
"""
-import os
-import sys
-from distutils.util import get_platform
-PLAT_SPEC = "%s-%d.%d" % (get_platform(), *sys.version_info[:2])
-src = os.path.join("build", "lib.%s" % PLAT_SPEC)
-sys.path.append(src)
-
if __name__ == "__main__":
import doctest, __main__
doctest.testmod(__main__)
diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py
deleted file mode 100644
index a38a39de3e7c86c..000000000000000
--- a/Doc/includes/setup.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from distutils.core import setup, Extension
-setup(name="noddy", version="1.0",
- ext_modules=[
- Extension("noddy", ["noddy.c"]),
- Extension("noddy2", ["noddy2.c"]),
- Extension("noddy3", ["noddy3.c"]),
- Extension("noddy4", ["noddy4.c"]),
- Extension("shoddy", ["shoddy.c"]),
- ])
diff --git a/Doc/includes/turtle-star.py b/Doc/includes/turtle-star.py
deleted file mode 100644
index 1a5db761b32385d..000000000000000
--- a/Doc/includes/turtle-star.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from turtle import *
-color('red', 'yellow')
-begin_fill()
-while True:
- forward(200)
- left(170)
- if abs(pos()) < 1:
- break
-end_fill()
-done()
diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h
index 02f8ccfe4438a5a..ec939c28831c33c 100644
--- a/Doc/includes/typestruct.h
+++ b/Doc/includes/typestruct.h
@@ -80,4 +80,7 @@ typedef struct _typeobject {
destructor tp_finalize;
vectorcallfunc tp_vectorcall;
+
+ /* bitset of which type-watchers care about this type */
+ unsigned char tp_watched;
} PyTypeObject;
diff --git a/Doc/install/index.rst b/Doc/install/index.rst
deleted file mode 100644
index ab581d785ef7f04..000000000000000
--- a/Doc/install/index.rst
+++ /dev/null
@@ -1,1081 +0,0 @@
-.. highlight:: none
-
-.. _install-index:
-
-********************************************
- Installing Python Modules (Legacy version)
-********************************************
-
-:Author: Greg Ward
-
-.. TODO: Fill in XXX comments
-
-.. note::
-
- The entire ``distutils`` package has been deprecated and will be
- removed in Python 3.12. This documentation is retained as a
- reference only, and will be removed with the package. See the
- :ref:`What's New ` entry for more information.
-
-.. seealso::
-
- :ref:`installing-index`
- The up to date module installation documentation. For regular Python
- usage, you almost certainly want that document rather than this one.
-
-.. note::
-
- This document is being retained solely until the ``setuptools`` documentation
- at https://setuptools.readthedocs.io/en/latest/setuptools.html
- independently covers all of the relevant information currently included here.
-
-.. note::
-
- This guide only covers the basic tools for building and distributing
- extensions that are provided as part of this version of Python. Third party
- tools offer easier to use and more secure alternatives. Refer to the `quick
- recommendations section `__
- in the Python Packaging User Guide for more information.
-
-
-.. _inst-intro:
-
-
-Introduction
-============
-
-In Python 2.0, the ``distutils`` API was first added to the standard library.
-This provided Linux distro maintainers with a standard way of converting
-Python projects into Linux distro packages, and system administrators with a
-standard way of installing them directly onto target systems.
-
-In the many years since Python 2.0 was released, tightly coupling the build
-system and package installer to the language runtime release cycle has turned
-out to be problematic, and it is now recommended that projects use the
-``pip`` package installer and the ``setuptools`` build system, rather than
-using ``distutils`` directly.
-
-See :ref:`installing-index` and :ref:`distributing-index` for more details.
-
-This legacy documentation is being retained only until we're confident that the
-``setuptools`` documentation covers everything needed.
-
-.. _inst-new-standard:
-
-Distutils based source distributions
-------------------------------------
-
-If you download a module source distribution, you can tell pretty quickly if it
-was packaged and distributed in the standard way, i.e. using the Distutils.
-First, the distribution's name and version number will be featured prominently
-in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or
-:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly named
-directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the
-distribution will contain a setup script :file:`setup.py`, and a file named
-:file:`README.txt` or possibly just :file:`README`, which should explain that
-building and installing the module distribution is a simple matter of running
-one command from a terminal::
-
- python setup.py install
-
-For Windows, this command should be run from a command prompt window
-(:menuselection:`Start --> Accessories`)::
-
- setup.py install
-
-If all these things are true, then you already know how to build and install the
-modules you've just downloaded: Run the command above. Unless you need to
-install things in a non-standard way or customize the build process, you don't
-really need this manual. Or rather, the above command is everything you need to
-get out of this manual.
-
-
-.. _inst-standard-install:
-
-Standard Build and Install
-==========================
-
-As described in section :ref:`inst-new-standard`, building and installing a module
-distribution using the Distutils is usually one simple command to run from a
-terminal::
-
- python setup.py install
-
-
-.. _inst-platform-variations:
-
-Platform variations
--------------------
-
-You should always run the setup command from the distribution root directory,
-i.e. the top-level subdirectory that the module source distribution unpacks
-into. For example, if you've just downloaded a module source distribution
-:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is::
-
- gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0
- cd foo-1.0
- python setup.py install
-
-On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the
-archive file to :file:`C:\\Temp`, then it would unpack into
-:file:`C:\\Temp\\foo-1.0`; you can use either an archive manipulator with a
-graphical user interface (such as WinZip) or a command-line tool (such as
-:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a
-command prompt window and run::
-
- cd c:\Temp\foo-1.0
- python setup.py install
-
-
-.. _inst-splitting-up:
-
-Splitting the job up
---------------------
-
-Running ``setup.py install`` builds and installs all modules in one run. If you
-prefer to work incrementally---especially useful if you want to customize the
-build process, or if things are going wrong---you can use the setup script to do
-one thing at a time. This is particularly helpful when the build and install
-will be done by different users---for example, you might want to build a module
-distribution and hand it off to a system administrator for installation (or do
-it yourself, with super-user privileges).
-
-For example, you can build everything in one step, and then install everything
-in a second step, by invoking the setup script twice::
-
- python setup.py build
- python setup.py install
-
-If you do this, you will notice that running the :command:`install` command
-first runs the :command:`build` command, which---in this case---quickly notices
-that it has nothing to do, since everything in the :file:`build` directory is
-up-to-date.
-
-You may not need this ability to break things down often if all you do is
-install modules downloaded off the 'net, but it's very handy for more advanced
-tasks. If you get into distributing your own Python modules and extensions,
-you'll run lots of individual Distutils commands on their own.
-
-
-.. _inst-how-build-works:
-
-How building works
-------------------
-
-As implied above, the :command:`build` command is responsible for putting the
-files to install into a *build directory*. By default, this is :file:`build`
-under the distribution root; if you're excessively concerned with speed, or want
-to keep the source tree pristine, you can change the build directory with the
-:option:`!--build-base` option. For example::
-
- python setup.py build --build-base=/path/to/pybuild/foo-1.0
-
-(Or you could do this permanently with a directive in your system or personal
-Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this
-isn't necessary.
-
-The default layout for the build tree is as follows::
-
- --- build/ --- lib/
- or
- --- build/ --- lib./
- temp./
-
-where ```` expands to a brief description of the current OS/hardware
-platform and Python version. The first form, with just a :file:`lib` directory,
-is used for "pure module distributions"---that is, module distributions that
-include only pure Python modules. If a module distribution contains any
-extensions (modules written in C/C++), then the second form, with two ````
-directories, is used. In that case, the :file:`temp.{plat}` directory holds
-temporary files generated by the compile/link process that don't actually get
-installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory
-contains all Python modules (pure Python and extensions) that will be installed.
-
-In the future, more directories will be added to handle Python scripts,
-documentation, binary executables, and whatever else is needed to handle the job
-of installing Python modules and applications.
-
-
-.. _inst-how-install-works:
-
-How installation works
-----------------------
-
-After the :command:`build` command runs (whether you run it explicitly, or the
-:command:`install` command does it for you), the work of the :command:`install`
-command is relatively simple: all it has to do is copy everything under
-:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation
-directory.
-
-If you don't choose an installation directory---i.e., if you just run ``setup.py
-install``\ ---then the :command:`install` command installs to the standard
-location for third-party Python modules. This location varies by platform and
-by how you built/installed Python itself. On Unix (and macOS, which is also
-Unix-based), it also depends on whether the module distribution being installed
-is pure Python or contains extensions ("non-pure"):
-
-.. tabularcolumns:: |l|l|l|l|
-
-+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
-| Platform | Standard installation location | Default value | Notes |
-+=================+=====================================================+==================================================+=======+
-| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) |
-+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
-| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) |
-+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
-| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) |
-+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
-
-Notes:
-
-(1)
- Most Linux distributions include Python as a standard part of the system, so
- :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on
- Linux. If you build Python yourself on Linux (or any Unix-like system), the
- default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`.
-
-(2)
- The default installation directory on Windows was :file:`C:\\Program
- Files\\Python` under Python 1.6a1, 1.5.2, and earlier.
-
-:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python
-is installed to, and where it finds its libraries at run-time. They are always
-the same under Windows, and very often the same under Unix and macOS. You
-can find out what your Python installation uses for :file:`{prefix}` and
-:file:`{exec-prefix}` by running Python in interactive mode and typing a few
-simple commands. Under Unix, just type ``python`` at the shell prompt. Under
-Windows, choose :menuselection:`Start --> Programs --> Python X.Y -->
-Python (command line)`. Once the interpreter is started, you type Python code
-at the prompt. For example, on my Linux system, I type the three Python
-statements shown below, and get the output as shown, to find out my
-:file:`{prefix}` and :file:`{exec-prefix}`:
-
-.. code-block:: pycon
-
- Python 2.4 (#26, Aug 7 2004, 17:19:02)
- Type "help", "copyright", "credits" or "license" for more information.
- >>> import sys
- >>> sys.prefix
- '/usr'
- >>> sys.exec_prefix
- '/usr'
-
-A few other placeholders are used in this document: :file:`{X.Y}` stands for the
-version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by
-the value of :data:`sys.abiflags` or the empty string for platforms which don't
-define ABI flags; :file:`{distname}` will be replaced by the name of the module
-distribution being installed. Dots and capitalization are important in the
-paths; for example, a value that uses ``python3.2`` on UNIX will typically use
-``Python32`` on Windows.
-
-If you don't want to install modules to the standard location, or if you don't
-have permission to write there, then you need to read about alternate
-installations in section :ref:`inst-alt-install`. If you want to customize your
-installation directories more heavily, see section :ref:`inst-custom-install` on
-custom installations.
-
-
-.. _inst-alt-install:
-
-Alternate Installation
-======================
-
-Often, it is necessary or desirable to install modules to a location other than
-the standard location for third-party Python modules. For example, on a Unix
-system you might not have permission to write to the standard third-party module
-directory. Or you might wish to try out a module before making it a standard
-part of your local Python installation. This is especially true when upgrading
-a distribution already present: you want to make sure your existing base of
-scripts still works with the new version before actually upgrading.
-
-The Distutils :command:`install` command is designed to make installing module
-distributions to an alternate location simple and painless. The basic idea is
-that you supply a base directory for the installation, and the
-:command:`install` command picks a set of directories (called an *installation
-scheme*) under this base directory in which to install files. The details
-differ across platforms, so read whichever of the following sections applies to
-you.
-
-Note that the various alternate installation schemes are mutually exclusive: you
-can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or
-``--install-base`` and ``--install-platbase``, but you can't mix from these
-groups.
-
-
-.. _inst-alt-install-user:
-
-Alternate installation: the user scheme
----------------------------------------
-
-This scheme is designed to be the most convenient solution for users that don't
-have write permission to the global site-packages directory or don't want to
-install into it. It is enabled with a simple option::
-
- python setup.py install --user
-
-Files will be installed into subdirectories of :data:`site.USER_BASE` (written
-as :file:`{userbase}` hereafter). This scheme installs pure Python modules and
-extension modules in the same location (also known as :data:`site.USER_SITE`).
-Here are the values for UNIX, including macOS:
-
-=============== ===========================================================
-Type of file Installation directory
-=============== ===========================================================
-modules :file:`{userbase}/lib/python{X.Y}/site-packages`
-scripts :file:`{userbase}/bin`
-data :file:`{userbase}`
-C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}`
-=============== ===========================================================
-
-And here are the values used on Windows:
-
-=============== ===========================================================
-Type of file Installation directory
-=============== ===========================================================
-modules :file:`{userbase}\\Python{XY}\\site-packages`
-scripts :file:`{userbase}\\Python{XY}\\Scripts`
-data :file:`{userbase}`
-C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}`
-=============== ===========================================================
-
-The advantage of using this scheme compared to the other ones described below is
-that the user site-packages directory is under normal conditions always included
-in :data:`sys.path` (see :mod:`site` for more information), which means that
-there is no additional step to perform after running the :file:`setup.py` script
-to finalize the installation.
-
-The :command:`build_ext` command also has a ``--user`` option to add
-:file:`{userbase}/include` to the compiler search path for header files and
-:file:`{userbase}/lib` to the compiler search path for libraries as well as to
-the runtime search path for shared C libraries (rpath).
-
-
-.. _inst-alt-install-home:
-
-Alternate installation: the home scheme
----------------------------------------
-
-The idea behind the "home scheme" is that you build and maintain a personal
-stash of Python modules. This scheme's name is derived from the idea of a
-"home" directory on Unix, since it's not unusual for a Unix user to make their
-home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`.
-This scheme can be used by anyone, regardless of the operating system they
-are installing for.
-
-Installing a new module distribution is as simple as ::
-
- python setup.py install --home=
-
-where you can supply any directory you like for the :option:`!--home` option. On
-Unix, lazy typists can just type a tilde (``~``); the :command:`install` command
-will expand this to your home directory::
-
- python setup.py install --home=~
-
-To make Python find the distributions installed with this scheme, you may have
-to :ref:`modify Python's search path ` or edit
-:mod:`sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit
-:data:`sys.path`.
-
-The :option:`!--home` option defines the installation base directory. Files are
-installed to the following directories under the installation base as follows:
-
-=============== ===========================================================
-Type of file Installation directory
-=============== ===========================================================
-modules :file:`{home}/lib/python`
-scripts :file:`{home}/bin`
-data :file:`{home}`
-C headers :file:`{home}/include/python/{distname}`
-=============== ===========================================================
-
-(Mentally replace slashes with backslashes if you're on Windows.)
-
-
-.. _inst-alt-install-prefix-unix:
-
-Alternate installation: Unix (the prefix scheme)
-------------------------------------------------
-
-The "prefix scheme" is useful when you wish to use one Python installation to
-perform the build/install (i.e., to run the setup script), but install modules
-into the third-party module directory of a different Python installation (or
-something that looks like a different Python installation). If this sounds a
-trifle unusual, it is---that's why the user and home schemes come before. However,
-there are at least two known cases where the prefix scheme will be useful.
-
-First, consider that many Linux distributions put Python in :file:`/usr`, rather
-than the more traditional :file:`/usr/local`. This is entirely appropriate,
-since in those cases Python is part of "the system" rather than a local add-on.
-However, if you are installing Python modules from source, you probably want
-them to go in :file:`/usr/local/lib/python2.{X}` rather than
-:file:`/usr/lib/python2.{X}`. This can be done with ::
-
- /usr/bin/python setup.py install --prefix=/usr/local
-
-Another possibility is a network filesystem where the name used to write to a
-remote directory is different from the name used to read it: for example, the
-Python interpreter accessed as :file:`/usr/local/bin/python` might search for
-modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to
-be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could
-be done with ::
-
- /usr/local/bin/python setup.py install --prefix=/mnt/@server/export
-
-In either case, the :option:`!--prefix` option defines the installation base, and
-the :option:`!--exec-prefix` option defines the platform-specific installation
-base, which is used for platform-specific files. (Currently, this just means
-non-pure module distributions, but could be expanded to C libraries, binary
-executables, etc.) If :option:`!--exec-prefix` is not supplied, it defaults to
-:option:`!--prefix`. Files are installed as follows:
-
-================= ==========================================================
-Type of file Installation directory
-================= ==========================================================
-Python modules :file:`{prefix}/lib/python{X.Y}/site-packages`
-extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages`
-scripts :file:`{prefix}/bin`
-data :file:`{prefix}`
-C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}`
-================= ==========================================================
-
-There is no requirement that :option:`!--prefix` or :option:`!--exec-prefix`
-actually point to an alternate Python installation; if the directories listed
-above do not already exist, they are created at installation time.
-
-Incidentally, the real reason the prefix scheme is important is simply that a
-standard Unix installation uses the prefix scheme, but with :option:`!--prefix`
-and :option:`!--exec-prefix` supplied by Python itself as ``sys.prefix`` and
-``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme,
-but every time you run ``python setup.py install`` without any other options,
-you're using it.
-
-Note that installing extensions to an alternate Python installation has no
-effect on how those extensions are built: in particular, the Python header files
-(:file:`Python.h` and friends) installed with the Python interpreter used to run
-the setup script will be used in compiling extensions. It is your
-responsibility to ensure that the interpreter used to run extensions installed
-in this way is compatible with the interpreter used to build them. The best way
-to do this is to ensure that the two interpreters are the same version of Python
-(possibly different builds, or possibly copies of the same build). (Of course,
-if your :option:`!--prefix` and :option:`!--exec-prefix` don't even point to an
-alternate Python installation, this is immaterial.)
-
-
-.. _inst-alt-install-prefix-windows:
-
-Alternate installation: Windows (the prefix scheme)
----------------------------------------------------
-
-Windows has no concept of a user's home directory, and since the standard Python
-installation under Windows is simpler than under Unix, the :option:`!--prefix`
-option has traditionally been used to install additional packages in separate
-locations on Windows. ::
-
- python setup.py install --prefix="\Temp\Python"
-
-to install modules to the :file:`\\Temp\\Python` directory on the current drive.
-
-The installation base is defined by the :option:`!--prefix` option; the
-:option:`!--exec-prefix` option is not supported under Windows, which means that
-pure Python modules and extension modules are installed into the same location.
-Files are installed as follows:
-
-=============== ==========================================================
-Type of file Installation directory
-=============== ==========================================================
-modules :file:`{prefix}\\Lib\\site-packages`
-scripts :file:`{prefix}\\Scripts`
-data :file:`{prefix}`
-C headers :file:`{prefix}\\Include\\{distname}`
-=============== ==========================================================
-
-
-.. _inst-custom-install:
-
-Custom Installation
-===================
-
-Sometimes, the alternate installation schemes described in section
-:ref:`inst-alt-install` just don't do what you want. You might want to tweak just
-one or two directories while keeping everything under the same base directory,
-or you might want to completely redefine the installation scheme. In either
-case, you're creating a *custom installation scheme*.
-
-To create a custom installation scheme, you start with one of the alternate
-schemes and override some of the installation directories used for the various
-types of files, using these options:
-
-====================== =======================
-Type of file Override option
-====================== =======================
-Python modules ``--install-purelib``
-extension modules ``--install-platlib``
-all modules ``--install-lib``
-scripts ``--install-scripts``
-data ``--install-data``
-C headers ``--install-headers``
-====================== =======================
-
-These override options can be relative, absolute,
-or explicitly defined in terms of one of the installation base directories.
-(There are two installation base directories, and they are normally the
-same---they only differ when you use the Unix "prefix scheme" and supply
-different ``--prefix`` and ``--exec-prefix`` options; using ``--install-lib``
-will override values computed or given for ``--install-purelib`` and
-``--install-platlib``, and is recommended for schemes that don't make a
-difference between Python and extension modules.)
-
-For example, say you're installing a module distribution to your home directory
-under Unix---but you want scripts to go in :file:`~/scripts` rather than
-:file:`~/bin`. As you might expect, you can override this directory with the
-:option:`!--install-scripts` option; in this case, it makes most sense to supply
-a relative path, which will be interpreted relative to the installation base
-directory (your home directory, in this case)::
-
- python setup.py install --home=~ --install-scripts=scripts
-
-Another Unix example: suppose your Python installation was built and installed
-with a prefix of :file:`/usr/local/python`, so under a standard installation
-scripts will wind up in :file:`/usr/local/python/bin`. If you want them in
-:file:`/usr/local/bin` instead, you would supply this absolute directory for the
-:option:`!--install-scripts` option::
-
- python setup.py install --install-scripts=/usr/local/bin
-
-(This performs an installation using the "prefix scheme", where the prefix is
-whatever your Python interpreter was installed with--- :file:`/usr/local/python`
-in this case.)
-
-If you maintain Python on Windows, you might want third-party modules to live in
-a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}`
-itself. This is almost as easy as customizing the script installation
-directory---you just have to remember that there are two types of modules
-to worry about, Python and extension modules, which can conveniently be both
-controlled by one option::
-
- python setup.py install --install-lib=Site
-
-The specified installation directory is relative to :file:`{prefix}`. Of
-course, you also have to ensure that this directory is in Python's module
-search path, such as by putting a :file:`.pth` file in a site directory (see
-:mod:`site`). See section :ref:`inst-search-path` to find out how to modify
-Python's search path.
-
-If you want to define an entire installation scheme, you just have to supply all
-of the installation directory options. The recommended way to do this is to
-supply relative paths; for example, if you want to maintain all Python
-module-related files under :file:`python` in your home directory, and you want a
-separate directory for each platform that you use your home directory from, you
-might define the following installation scheme::
-
- python setup.py install --home=~ \
- --install-purelib=python/lib \
- --install-platlib=python/lib.$PLAT \
- --install-scripts=python/scripts
- --install-data=python/data
-
-or, equivalently, ::
-
- python setup.py install --home=~/python \
- --install-purelib=lib \
- --install-platlib='lib.$PLAT' \
- --install-scripts=scripts
- --install-data=data
-
-``$PLAT`` is not (necessarily) an environment variable---it will be expanded by
-the Distutils as it parses your command line options, just as it does when
-parsing your configuration file(s).
-
-Obviously, specifying the entire installation scheme every time you install a
-new module distribution would be very tedious. Thus, you can put these options
-into your Distutils config file (see section :ref:`inst-config-files`):
-
-.. code-block:: ini
-
- [install]
- install-base=$HOME
- install-purelib=python/lib
- install-platlib=python/lib.$PLAT
- install-scripts=python/scripts
- install-data=python/data
-
-or, equivalently,
-
-.. code-block:: ini
-
- [install]
- install-base=$HOME/python
- install-purelib=lib
- install-platlib=lib.$PLAT
- install-scripts=scripts
- install-data=data
-
-Note that these two are *not* equivalent if you supply a different installation
-base directory when you run the setup script. For example, ::
-
- python setup.py install --install-base=/tmp
-
-would install pure modules to :file:`/tmp/python/lib` in the first case, and
-to :file:`/tmp/lib` in the second case. (For the second case, you probably
-want to supply an installation base of :file:`/tmp/python`.)
-
-You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample
-configuration file input. These are Distutils configuration variables, which
-bear a strong resemblance to environment variables. In fact, you can use
-environment variables in config files on platforms that have such a notion but
-the Distutils additionally define a few extra variables that may not be in your
-environment, such as ``$PLAT``. (And of course, on systems that don't have
-environment variables, such as Mac OS 9, the configuration variables supplied by
-the Distutils are the only ones you can use.) See section :ref:`inst-config-files`
-for details.
-
-.. note:: When a :ref:`virtual environment ` is activated, any options
- that change the installation path will be ignored from all distutils configuration
- files to prevent inadvertently installing projects outside of the virtual
- environment.
-
-.. XXX need some Windows examples---when would custom installation schemes be
- needed on those platforms?
-
-
-.. XXX Move this to Doc/using
-
-.. _inst-search-path:
-
-Modifying Python's Search Path
-------------------------------
-
-When the Python interpreter executes an :keyword:`import` statement, it searches
-for both Python code and extension modules along a search path. A default value
-for the path is configured into the Python binary when the interpreter is built.
-You can determine the path by importing the :mod:`sys` module and printing the
-value of ``sys.path``. ::
-
- $ python
- Python 2.2 (#11, Oct 3 2002, 13:31:27)
- [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2
- Type "help", "copyright", "credits" or "license" for more information.
- >>> import sys
- >>> sys.path
- ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2',
- '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload',
- '/usr/local/lib/python2.3/site-packages']
- >>>
-
-The null string in ``sys.path`` represents the current working directory.
-
-The expected convention for locally installed packages is to put them in the
-:file:`{...}/site-packages/` directory, but you may want to install Python
-modules into some arbitrary directory. For example, your site may have a
-convention of keeping all software related to the web server under :file:`/www`.
-Add-on Python modules might then belong in :file:`/www/python`, and in order to
-import them, this directory must be added to ``sys.path``. There are several
-different ways to add the directory.
-
-The most convenient way is to add a path configuration file to a directory
-that's already on Python's path, usually to the :file:`.../site-packages/`
-directory. Path configuration files have an extension of :file:`.pth`, and each
-line must contain a single path that will be appended to ``sys.path``. (Because
-the new paths are appended to ``sys.path``, modules in the added directories
-will not override standard modules. This means you can't use this mechanism for
-installing fixed versions of standard modules.)
-
-Paths can be absolute or relative, in which case they're relative to the
-directory containing the :file:`.pth` file. See the documentation of
-the :mod:`site` module for more information.
-
-A slightly less convenient way is to edit the :file:`site.py` file in Python's
-standard library, and modify ``sys.path``. :file:`site.py` is automatically
-imported when the Python interpreter is executed, unless the :option:`-S` switch
-is supplied to suppress this behaviour. So you could simply edit
-:file:`site.py` and add two lines to it:
-
-.. code-block:: python
-
- import sys
- sys.path.append('/www/python/')
-
-However, if you reinstall the same major version of Python (perhaps when
-upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by
-the stock version. You'd have to remember that it was modified and save a copy
-before doing the installation.
-
-There are two environment variables that can modify ``sys.path``.
-:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python
-installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``,
-the search path will be set to ``['', '/www/python/lib/pythonX.Y/',
-'/www/python/lib/pythonX.Y/plat-linux2', ...]``.
-
-The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be
-added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is
-set to ``/www/python:/opt/py``, the search path will begin with
-``['/www/python', '/opt/py']``. (Note that directories must exist in order to
-be added to ``sys.path``; the :mod:`site` module removes paths that don't
-exist.)
-
-Finally, ``sys.path`` is just a regular Python list, so any Python application
-can modify it by adding or removing entries.
-
-
-.. _inst-config-files:
-
-Distutils Configuration Files
-=============================
-
-As mentioned above, you can use Distutils configuration files to record personal
-or site preferences for any Distutils options. That is, any option to any
-command can be stored in one of two or three (depending on your platform)
-configuration files, which will be consulted before the command-line is parsed.
-This means that configuration files will override default values, and the
-command-line will in turn override configuration files. Furthermore, if
-multiple configuration files apply, values from "earlier" files are overridden
-by "later" files.
-
-
-.. _inst-config-filenames:
-
-Location and names of config files
-----------------------------------
-
-The names and locations of the configuration files vary slightly across
-platforms. On Unix and macOS, the three configuration files (in the order
-they are processed) are:
-
-+--------------+----------------------------------------------------------+-------+
-| Type of file | Location and filename | Notes |
-+==============+==========================================================+=======+
-| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) |
-+--------------+----------------------------------------------------------+-------+
-| personal | :file:`$HOME/.pydistutils.cfg` | \(2) |
-+--------------+----------------------------------------------------------+-------+
-| local | :file:`setup.cfg` | \(3) |
-+--------------+----------------------------------------------------------+-------+
-
-And on Windows, the configuration files are:
-
-+--------------+-------------------------------------------------+-------+
-| Type of file | Location and filename | Notes |
-+==============+=================================================+=======+
-| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) |
-+--------------+-------------------------------------------------+-------+
-| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) |
-+--------------+-------------------------------------------------+-------+
-| local | :file:`setup.cfg` | \(3) |
-+--------------+-------------------------------------------------+-------+
-
-On all platforms, the "personal" file can be temporarily disabled by
-passing the ``--no-user-cfg`` option.
-
-Notes:
-
-(1)
- Strictly speaking, the system-wide configuration file lives in the directory
- where the Distutils are installed; under Python 1.6 and later on Unix, this is
- as shown. For Python 1.5.2, the Distutils will normally be installed to
- :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system
- configuration file should be put there under Python 1.5.2.
-
-(2)
- On Unix, if the :envvar:`HOME` environment variable is not defined, the user's
- home directory will be determined with the :func:`getpwuid` function from the
- standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser`
- function used by Distutils.
-
-(3)
- I.e., in the current directory (usually the location of the setup script).
-
-(4)
- (See also note (1).) Under Python 1.6 and later, Python's default "installation
- prefix" is :file:`C:\\Python`, so the system configuration file is normally
- :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the
- default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not
- part of the standard library---so the system configuration file would be
- :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python
- 1.5.2 installation under Windows.
-
-(5)
- On Windows, if the :envvar:`HOME` environment variable is not defined,
- :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will
- be tried. This is done by the :func:`os.path.expanduser` function used
- by Distutils.
-
-
-.. _inst-config-syntax:
-
-Syntax of config files
-----------------------
-
-The Distutils configuration files all have the same syntax. The config files
-are grouped into sections. There is one section for each Distutils command,
-plus a ``global`` section for global options that affect every command. Each
-section consists of one option per line, specified as ``option=value``.
-
-For example, the following is a complete config file that just forces all
-commands to run quietly by default:
-
-.. code-block:: ini
-
- [global]
- verbose=0
-
-If this is installed as the system config file, it will affect all processing of
-any Python module distribution by any user on the current system. If it is
-installed as your personal config file (on systems that support them), it will
-affect only module distributions processed by you. And if it is used as the
-:file:`setup.cfg` for a particular module distribution, it affects only that
-distribution.
-
-You could override the default "build base" directory and make the
-:command:`build\*` commands always forcibly rebuild all files with the
-following:
-
-.. code-block:: ini
-
- [build]
- build-base=blib
- force=1
-
-which corresponds to the command-line arguments ::
-
- python setup.py build --build-base=blib --force
-
-except that including the :command:`build` command on the command-line means
-that command will be run. Including a particular command in config files has no
-such implication; it only means that if the command is run, the options in the
-config file will apply. (Or if other commands that derive values from it are
-run, they will use the values in the config file.)
-
-You can find out the complete list of options for any command using the
-:option:`!--help` option, e.g.::
-
- python setup.py build --help
-
-and you can find out the complete list of global options by using
-:option:`!--help` without a command::
-
- python setup.py --help
-
-See also the "Reference" section of the "Distributing Python Modules" manual.
-
-
-.. _inst-building-ext:
-
-Building Extensions: Tips and Tricks
-====================================
-
-Whenever possible, the Distutils try to use the configuration information made
-available by the Python interpreter used to run the :file:`setup.py` script.
-For example, the same compiler and linker flags used to compile Python will also
-be used for compiling extensions. Usually this will work well, but in
-complicated situations this might be inappropriate. This section discusses how
-to override the usual Distutils behaviour.
-
-
-.. _inst-tweak-flags:
-
-Tweaking compiler/linker flags
-------------------------------
-
-Compiling a Python extension written in C or C++ will sometimes require
-specifying custom flags for the compiler and linker in order to use a particular
-library or produce a special kind of object code. This is especially true if the
-extension hasn't been tested on your platform, or if you're trying to
-cross-compile Python.
-
-In the most general case, the extension author might have foreseen that
-compiling the extensions would be complicated, and provided a :file:`Setup` file
-for you to edit. This will likely only be done if the module distribution
-contains many separate extension modules, or if they often require elaborate
-sets of compiler flags in order to work.
-
-A :file:`Setup` file, if present, is parsed in order to get a list of extensions
-to build. Each line in a :file:`Setup` describes a single module. Lines have
-the following structure::
-
- module ... [sourcefile ...] [cpparg ...] [library ...]
-
-
-Let's examine each of the fields in turn.
-
-* *module* is the name of the extension module to be built, and should be a
- valid Python identifier. You can't just change this in order to rename a module
- (edits to the source code would also be needed), so this should be left alone.
-
-* *sourcefile* is anything that's likely to be a source code file, at least
- judging by the filename. Filenames ending in :file:`.c` are assumed to be
- written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are
- assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed
- to be in Objective C.
-
-* *cpparg* is an argument for the C preprocessor, and is anything starting with
- :option:`!-I`, :option:`!-D`, :option:`!-U` or :option:`!-C`.
-
-* *library* is anything ending in :file:`.a` or beginning with :option:`!-l` or
- :option:`!-L`.
-
-If a particular platform requires a special library on your platform, you can
-add it by editing the :file:`Setup` file and running ``python setup.py build``.
-For example, if the module defined by the line ::
-
- foo foomodule.c
-
-must be linked with the math library :file:`libm.a` on your platform, simply add
-:option:`!-lm` to the line::
-
- foo foomodule.c -lm
-
-Arbitrary switches intended for the compiler or the linker can be supplied with
-the :option:`!-Xcompiler` *arg* and :option:`!-Xlinker` *arg* options::
-
- foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm
-
-The next option after :option:`!-Xcompiler` and :option:`!-Xlinker` will be
-appended to the proper command line, so in the above example the compiler will
-be passed the :option:`!-o32` option, and the linker will be passed
-:option:`!-shared`. If a compiler option requires an argument, you'll have to
-supply multiple :option:`!-Xcompiler` options; for example, to pass ``-x c++``
-the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``.
-
-Compiler flags can also be supplied through setting the :envvar:`CFLAGS`
-environment variable. If set, the contents of :envvar:`CFLAGS` will be added to
-the compiler flags specified in the :file:`Setup` file.
-
-
-.. _inst-non-ms-compilers:
-
-Using non-Microsoft compilers on Windows
-----------------------------------------
-
-.. sectionauthor:: Rene Liebscher
-
-
-
-Borland/CodeGear C++
-^^^^^^^^^^^^^^^^^^^^
-
-This subsection describes the necessary steps to use Distutils with the Borland
-C++ compiler version 5.5. First you have to know that Borland's object file
-format (OMF) is different from the format used by the Python version you can
-download from the Python or ActiveState web site. (Python is built with
-Microsoft Visual C++, which uses COFF as the object file format.) For this
-reason you have to convert Python's library :file:`python25.lib` into the
-Borland format. You can do this as follows:
-
-.. Should we mention that users have to create cfg-files for the compiler?
-.. see also http://community.borland.com/article/0,1410,21205,00.html
-
-::
-
- coff2omf python25.lib python25_bcpp.lib
-
-The :file:`coff2omf` program comes with the Borland compiler. The file
-:file:`python25.lib` is in the :file:`Libs` directory of your Python
-installation. If your extension uses other libraries (zlib, ...) you have to
-convert them too.
-
-The converted files have to reside in the same directories as the normal
-libraries.
-
-How does Distutils manage to use these libraries with their changed names? If
-the extension needs a library (eg. :file:`foo`) Distutils checks first if it
-finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then
-uses this library. In the case it doesn't find such a special library it uses
-the default name (:file:`foo.lib`.) [#]_
-
-To let Distutils compile your extension with Borland C++ you now have to type::
-
- python setup.py build --compiler=bcpp
-
-If you want to use the Borland C++ compiler as the default, you could specify
-this in your personal or system-wide configuration file for Distutils (see
-section :ref:`inst-config-files`.)
-
-
-.. seealso::
-
- `C++Builder Compiler `_
- Information about the free C++ compiler from Borland, including links to the
- download pages.
-
- `Creating Python Extensions Using Borland's Free Compiler `_
- Document describing how to use Borland's free command-line C++ compiler to build
- Python.
-
-
-GNU C / Cygwin / MinGW
-^^^^^^^^^^^^^^^^^^^^^^
-
-This section describes the necessary steps to use Distutils with the GNU C/C++
-compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter
-that was built with Cygwin, everything should work without any of these
-following steps.
-
-Not all extensions can be built with MinGW or Cygwin, but many can. Extensions
-most likely to not work are those that use C++ or depend on Microsoft Visual C
-extensions.
-
-To let Distutils compile your extension with Cygwin you have to type::
-
- python setup.py build --compiler=cygwin
-
-and for Cygwin in no-cygwin mode [#]_ or for MinGW type::
-
- python setup.py build --compiler=mingw32
-
-If you want to use any of these options/compilers as default, you should
-consider writing it in your personal or system-wide configuration file for
-Distutils (see section :ref:`inst-config-files`.)
-
-Older Versions of Python and MinGW
-""""""""""""""""""""""""""""""""""
-The following instructions only apply if you're using a version of Python
-inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with
-binutils-2.13.90-20030111-1).
-
-These compilers require some special libraries. This task is more complex than
-for Borland's C++, because there is no program to convert the library. First
-you have to create a list of symbols which the Python DLL exports. (You can find
-a good program for this task at
-https://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/).
-
-.. I don't understand what the next line means. --amk
-.. (inclusive the references on data structures.)
-
-::
-
- pexports python25.dll >python25.def
-
-The location of an installed :file:`python25.dll` will depend on the
-installation options and the version and language of Windows. In a "just for
-me" installation, it will appear in the root of the installation directory. In
-a shared installation, it will be located in the system directory.
-
-Then you can create from these information an import library for gcc. ::
-
- /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a
-
-The resulting library has to be placed in the same directory as
-:file:`python25.lib`. (Should be the :file:`libs` directory under your Python
-installation directory.)
-
-If your extension uses other libraries (zlib,...) you might have to convert
-them too. The converted files have to reside in the same directories as the
-normal libraries do.
-
-
-.. seealso::
-
- `Building Python modules on MS Windows platform with MinGW `_
- Information about building the required libraries for the MinGW environment.
-
-
-.. rubric:: Footnotes
-
-.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries
- of the same name.
-
-.. [#] Check https://www.sourceware.org/cygwin/ for more information
-
-.. [#] Then you have no POSIX emulation available, but you also don't need
- :file:`cygwin1.dll`.
diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst
index e158bf1c4c0c7f5..a46c1caefe4d8ac 100644
--- a/Doc/installing/index.rst
+++ b/Doc/installing/index.rst
@@ -19,7 +19,9 @@ solutions to the common pool.
This guide covers the installation part of the process. For a guide to
creating and sharing your own Python projects, refer to the
-:ref:`distribution guide `.
+`Python packaging user guide`_.
+
+.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/
.. note::
@@ -52,8 +54,7 @@ Key terms
developers and documentation authors responsible for the maintenance and
evolution of the standard packaging tools and the associated metadata and
file format standards. They maintain a variety of tools, documentation,
- and issue trackers on both `GitHub `__ and
- `Bitbucket `__.
+ and issue trackers on `GitHub `__.
* ``distutils`` is the original build and distribution system first added to
the Python standard library in 1998. While direct use of ``distutils`` is
being phased out, it still laid the foundation for the current packaging
diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst
deleted file mode 100644
index d85ad94e9b7fe47..000000000000000
--- a/Doc/library/2to3.rst
+++ /dev/null
@@ -1,489 +0,0 @@
-.. _2to3-reference:
-
-2to3 --- Automated Python 2 to 3 code translation
-=================================================
-
-.. sectionauthor:: Benjamin Peterson
-
-2to3 is a Python program that reads Python 2.x source code and applies a series
-of *fixers* to transform it into valid Python 3.x code. The standard library
-contains a rich set of fixers that will handle almost all code. 2to3 supporting
-library :mod:`lib2to3` is, however, a flexible and generic library, so it is
-possible to write your own fixers for 2to3.
-
-.. deprecated-removed:: 3.11 3.13
- The ``lib2to3`` module was marked pending for deprecation in Python 3.9
- (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated
- in Python 3.11 (raising :exc:`DeprecationWarning`). The ``2to3`` tool is
- part of that. It will be removed in Python 3.13.
-
-.. _2to3-using:
-
-Using 2to3
-----------
-
-2to3 will usually be installed with the Python interpreter as a script. It is
-also located in the :file:`Tools/scripts` directory of the Python root.
-
-2to3's basic arguments are a list of files or directories to transform. The
-directories are recursively traversed for Python sources.
-
-Here is a sample Python 2.x source file, :file:`example.py`::
-
- def greet(name):
- print "Hello, {0}!".format(name)
- print "What's your name?"
- name = raw_input()
- greet(name)
-
-It can be converted to Python 3.x code via 2to3 on the command line:
-
-.. code-block:: shell-session
-
- $ 2to3 example.py
-
-A diff against the original source file is printed. 2to3 can also write the
-needed modifications right back to the source file. (A backup of the original
-file is made unless :option:`!-n` is also given.) Writing the changes back is
-enabled with the :option:`!-w` flag:
-
-.. code-block:: shell-session
-
- $ 2to3 -w example.py
-
-After transformation, :file:`example.py` looks like this::
-
- def greet(name):
- print("Hello, {0}!".format(name))
- print("What's your name?")
- name = input()
- greet(name)
-
-Comments and exact indentation are preserved throughout the translation process.
-
-By default, 2to3 runs a set of :ref:`predefined fixers <2to3-fixers>`. The
-:option:`!-l` flag lists all available fixers. An explicit set of fixers to run
-can be given with :option:`!-f`. Likewise the :option:`!-x` explicitly disables a
-fixer. The following example runs only the ``imports`` and ``has_key`` fixers:
-
-.. code-block:: shell-session
-
- $ 2to3 -f imports -f has_key example.py
-
-This command runs every fixer except the ``apply`` fixer:
-
-.. code-block:: shell-session
-
- $ 2to3 -x apply example.py
-
-Some fixers are *explicit*, meaning they aren't run by default and must be
-listed on the command line to be run. Here, in addition to the default fixers,
-the ``idioms`` fixer is run:
-
-.. code-block:: shell-session
-
- $ 2to3 -f all -f idioms example.py
-
-Notice how passing ``all`` enables all default fixers.
-
-Sometimes 2to3 will find a place in your source code that needs to be changed,
-but 2to3 cannot fix automatically. In this case, 2to3 will print a warning
-beneath the diff for a file. You should address the warning in order to have
-compliant 3.x code.
-
-2to3 can also refactor doctests. To enable this mode, use the :option:`!-d`
-flag. Note that *only* doctests will be refactored. This also doesn't require
-the module to be valid Python. For example, doctest like examples in a reST
-document could also be refactored with this option.
-
-The :option:`!-v` option enables output of more information on the translation
-process.
-
-Since some print statements can be parsed as function calls or statements, 2to3
-cannot always read files containing the print function. When 2to3 detects the
-presence of the ``from __future__ import print_function`` compiler directive, it
-modifies its internal grammar to interpret :func:`print` as a function. This
-change can also be enabled manually with the :option:`!-p` flag. Use
-:option:`!-p` to run fixers on code that already has had its print statements
-converted. Also :option:`!-e` can be used to make :func:`exec` a function.
-
-The :option:`!-o` or :option:`!--output-dir` option allows specification of an
-alternate directory for processed output files to be written to. The
-:option:`!-n` flag is required when using this as backup files do not make sense
-when not overwriting the input files.
-
-.. versionadded:: 3.2.3
- The :option:`!-o` option was added.
-
-The :option:`!-W` or :option:`!--write-unchanged-files` flag tells 2to3 to always
-write output files even if no changes were required to the file. This is most
-useful with :option:`!-o` so that an entire Python source tree is copied with
-translation from one directory to another.
-This option implies the :option:`!-w` flag as it would not make sense otherwise.
-
-.. versionadded:: 3.2.3
- The :option:`!-W` flag was added.
-
-The :option:`!--add-suffix` option specifies a string to append to all output
-filenames. The :option:`!-n` flag is required when specifying this as backups
-are not necessary when writing to different filenames. Example:
-
-.. code-block:: shell-session
-
- $ 2to3 -n -W --add-suffix=3 example.py
-
-Will cause a converted file named ``example.py3`` to be written.
-
-.. versionadded:: 3.2.3
- The :option:`!--add-suffix` option was added.
-
-To translate an entire project from one directory tree to another use:
-
-.. code-block:: shell-session
-
- $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode
-
-
-.. _2to3-fixers:
-
-Fixers
-------
-
-Each step of transforming code is encapsulated in a fixer. The command ``2to3
--l`` lists them. As :ref:`documented above <2to3-using>`, each can be turned on
-and off individually. They are described here in more detail.
-
-
-.. 2to3fixer:: apply
-
- Removes usage of :func:`apply`. For example ``apply(function, *args,
- **kwargs)`` is converted to ``function(*args, **kwargs)``.
-
-.. 2to3fixer:: asserts
-
- Replaces deprecated :mod:`unittest` method names with the correct ones.
-
- ================================ ==========================================
- From To
- ================================ ==========================================
- ``failUnlessEqual(a, b)`` :meth:`assertEqual(a, b)
- `
- ``assertEquals(a, b)`` :meth:`assertEqual(a, b)
- `
- ``failIfEqual(a, b)`` :meth:`assertNotEqual(a, b)
- `
- ``assertNotEquals(a, b)`` :meth:`assertNotEqual(a, b)
- `
- ``failUnless(a)`` :meth:`assertTrue(a)
- `
- ``assert_(a)`` :meth:`assertTrue(a)
- `
- ``failIf(a)`` :meth:`assertFalse(a)
- `
- ``failUnlessRaises(exc, cal)`` :meth:`assertRaises(exc, cal)
- `
- ``failUnlessAlmostEqual(a, b)`` :meth:`assertAlmostEqual(a, b)
- `
- ``assertAlmostEquals(a, b)`` :meth:`assertAlmostEqual(a, b)
- `
- ``failIfAlmostEqual(a, b)`` :meth:`assertNotAlmostEqual(a, b)
- `
- ``assertNotAlmostEquals(a, b)`` :meth:`assertNotAlmostEqual(a, b)
- `
- ================================ ==========================================
-
-.. 2to3fixer:: basestring
-
- Converts :class:`basestring` to :class:`str`.
-
-.. 2to3fixer:: buffer
-
- Converts :class:`buffer` to :class:`memoryview`. This fixer is optional
- because the :class:`memoryview` API is similar but not exactly the same as
- that of :class:`buffer`.
-
-.. 2to3fixer:: dict
-
- Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to
- :meth:`dict.items`, :meth:`dict.iterkeys` to :meth:`dict.keys`, and
- :meth:`dict.itervalues` to :meth:`dict.values`. Similarly,
- :meth:`dict.viewitems`, :meth:`dict.viewkeys` and :meth:`dict.viewvalues` are
- converted respectively to :meth:`dict.items`, :meth:`dict.keys` and
- :meth:`dict.values`. It also wraps existing usages of :meth:`dict.items`,
- :meth:`dict.keys`, and :meth:`dict.values` in a call to :class:`list`.
-
-.. 2to3fixer:: except
-
- Converts ``except X, T`` to ``except X as T``.
-
-.. 2to3fixer:: exec
-
- Converts the ``exec`` statement to the :func:`exec` function.
-
-.. 2to3fixer:: execfile
-
- Removes usage of :func:`execfile`. The argument to :func:`execfile` is
- wrapped in calls to :func:`open`, :func:`compile`, and :func:`exec`.
-
-.. 2to3fixer:: exitfunc
-
- Changes assignment of :attr:`sys.exitfunc` to use of the :mod:`atexit`
- module.
-
-.. 2to3fixer:: filter
-
- Wraps :func:`filter` usage in a :class:`list` call.
-
-.. 2to3fixer:: funcattrs
-
- Fixes function attributes that have been renamed. For example,
- ``my_function.func_closure`` is converted to ``my_function.__closure__``.
-
-.. 2to3fixer:: future
-
- Removes ``from __future__ import new_feature`` statements.
-
-.. 2to3fixer:: getcwdu
-
- Renames :func:`os.getcwdu` to :func:`os.getcwd`.
-
-.. 2to3fixer:: has_key
-
- Changes ``dict.has_key(key)`` to ``key in dict``.
-
-.. 2to3fixer:: idioms
-
- This optional fixer performs several transformations that make Python code
- more idiomatic. Type comparisons like ``type(x) is SomeClass`` and
- ``type(x) == SomeClass`` are converted to ``isinstance(x, SomeClass)``.
- ``while 1`` becomes ``while True``. This fixer also tries to make use of
- :func:`sorted` in appropriate places. For example, this block ::
-
- L = list(some_iterable)
- L.sort()
-
- is changed to ::
-
- L = sorted(some_iterable)
-
-.. 2to3fixer:: import
-
- Detects sibling imports and converts them to relative imports.
-
-.. 2to3fixer:: imports
-
- Handles module renames in the standard library.
-
-.. 2to3fixer:: imports2
-
- Handles other modules renames in the standard library. It is separate from
- the :2to3fixer:`imports` fixer only because of technical limitations.
-
-.. 2to3fixer:: input
-
- Converts ``input(prompt)`` to ``eval(input(prompt))``.
-
-.. 2to3fixer:: intern
-
- Converts :func:`intern` to :func:`sys.intern`.
-
-.. 2to3fixer:: isinstance
-
- Fixes duplicate types in the second argument of :func:`isinstance`. For
- example, ``isinstance(x, (int, int))`` is converted to ``isinstance(x,
- int)`` and ``isinstance(x, (int, float, int))`` is converted to
- ``isinstance(x, (int, float))``.
-
-.. 2to3fixer:: itertools_imports
-
- Removes imports of :func:`itertools.ifilter`, :func:`itertools.izip`, and
- :func:`itertools.imap`. Imports of :func:`itertools.ifilterfalse` are also
- changed to :func:`itertools.filterfalse`.
-
-.. 2to3fixer:: itertools
-
- Changes usage of :func:`itertools.ifilter`, :func:`itertools.izip`, and
- :func:`itertools.imap` to their built-in equivalents.
- :func:`itertools.ifilterfalse` is changed to :func:`itertools.filterfalse`.
-
-.. 2to3fixer:: long
-
- Renames :class:`long` to :class:`int`.
-
-.. 2to3fixer:: map
-
- Wraps :func:`map` in a :class:`list` call. It also changes ``map(None, x)``
- to ``list(x)``. Using ``from future_builtins import map`` disables this
- fixer.
-
-.. 2to3fixer:: metaclass
-
- Converts the old metaclass syntax (``__metaclass__ = Meta`` in the class
- body) to the new (``class X(metaclass=Meta)``).
-
-.. 2to3fixer:: methodattrs
-
- Fixes old method attribute names. For example, ``meth.im_func`` is converted
- to ``meth.__func__``.
-
-.. 2to3fixer:: ne
-
- Converts the old not-equal syntax, ``<>``, to ``!=``.
-
-.. 2to3fixer:: next
-
- Converts the use of iterator's :meth:`~iterator.next` methods to the
- :func:`next` function. It also renames :meth:`next` methods to
- :meth:`~iterator.__next__`.
-
-.. 2to3fixer:: nonzero
-
- Renames definitions of methods called :meth:`__nonzero__`
- to :meth:`~object.__bool__`.
-
-.. 2to3fixer:: numliterals
-
- Converts octal literals into the new syntax.
-
-.. 2to3fixer:: operator
-
- Converts calls to various functions in the :mod:`operator` module to other,
- but equivalent, function calls. When needed, the appropriate ``import``
- statements are added, e.g. ``import collections.abc``. The following mapping
- are made:
-
- ================================== =============================================
- From To
- ================================== =============================================
- ``operator.isCallable(obj)`` ``callable(obj)``
- ``operator.sequenceIncludes(obj)`` ``operator.contains(obj)``
- ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.abc.Sequence)``
- ``operator.isMappingType(obj)`` ``isinstance(obj, collections.abc.Mapping)``
- ``operator.isNumberType(obj)`` ``isinstance(obj, numbers.Number)``
- ``operator.repeat(obj, n)`` ``operator.mul(obj, n)``
- ``operator.irepeat(obj, n)`` ``operator.imul(obj, n)``
- ================================== =============================================
-
-.. 2to3fixer:: paren
-
- Add extra parenthesis where they are required in list comprehensions. For
- example, ``[x for x in 1, 2]`` becomes ``[x for x in (1, 2)]``.
-
-.. 2to3fixer:: print
-
- Converts the ``print`` statement to the :func:`print` function.
-
-.. 2to3fixer:: raise
-
- Converts ``raise E, V`` to ``raise E(V)``, and ``raise E, V, T`` to ``raise
- E(V).with_traceback(T)``. If ``E`` is a tuple, the translation will be
- incorrect because substituting tuples for exceptions has been removed in 3.0.
-
-.. 2to3fixer:: raw_input
-
- Converts :func:`raw_input` to :func:`input`.
-
-.. 2to3fixer:: reduce
-
- Handles the move of :func:`reduce` to :func:`functools.reduce`.
-
-.. 2to3fixer:: reload
-
- Converts :func:`reload` to :func:`importlib.reload`.
-
-.. 2to3fixer:: renames
-
- Changes :data:`sys.maxint` to :data:`sys.maxsize`.
-
-.. 2to3fixer:: repr
-
- Replaces backtick repr with the :func:`repr` function.
-
-.. 2to3fixer:: set_literal
-
- Replaces use of the :class:`set` constructor with set literals. This fixer
- is optional.
-
-.. 2to3fixer:: standarderror
-
- Renames :exc:`StandardError` to :exc:`Exception`.
-
-.. 2to3fixer:: sys_exc
-
- Changes the deprecated :data:`sys.exc_value`, :data:`sys.exc_type`,
- :data:`sys.exc_traceback` to use :func:`sys.exc_info`.
-
-.. 2to3fixer:: throw
-
- Fixes the API change in generator's :meth:`throw` method.
-
-.. 2to3fixer:: tuple_params
-
- Removes implicit tuple parameter unpacking. This fixer inserts temporary
- variables.
-
-.. 2to3fixer:: types
-
- Fixes code broken from the removal of some members in the :mod:`types`
- module.
-
-.. 2to3fixer:: unicode
-
- Renames :class:`unicode` to :class:`str`.
-
-.. 2to3fixer:: urllib
-
- Handles the rename of :mod:`urllib` and :mod:`urllib2` to the :mod:`urllib`
- package.
-
-.. 2to3fixer:: ws_comma
-
- Removes excess whitespace from comma separated items. This fixer is
- optional.
-
-.. 2to3fixer:: xrange
-
- Renames :func:`xrange` to :func:`range` and wraps existing :func:`range`
- calls with :class:`list`.
-
-.. 2to3fixer:: xreadlines
-
- Changes ``for x in file.xreadlines()`` to ``for x in file``.
-
-.. 2to3fixer:: zip
-
- Wraps :func:`zip` usage in a :class:`list` call. This is disabled when
- ``from future_builtins import zip`` appears.
-
-
-:mod:`lib2to3` --- 2to3's library
----------------------------------
-
-.. module:: lib2to3
- :synopsis: The 2to3 library
-
-.. moduleauthor:: Guido van Rossum
-.. moduleauthor:: Collin Winter
-.. moduleauthor:: Benjamin Peterson
-
-**Source code:** :source:`Lib/lib2to3/`
-
---------------
-
-.. deprecated-removed:: 3.11 3.13
- Python 3.9 switched to a PEG parser (see :pep:`617`) while lib2to3 is
- using a less flexible LL(1) parser. Python 3.10 includes new language
- syntax that is not parsable by lib2to3's LL(1) parser (see :pep:`634`).
- The ``lib2to3`` module was marked pending for deprecation in Python 3.9
- (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated
- in Python 3.11 (raising :exc:`DeprecationWarning`).
- It will be removed from the standard library in Python 3.13.
- Consider third-party alternatives such as `LibCST`_ or `parso`_.
-
-.. note::
-
- The :mod:`lib2to3` API should be considered unstable and may change
- drastically in the future.
-
-.. _LibCST: https://libcst.readthedocs.io/
-.. _parso: https://parso.readthedocs.io/
diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst
index 8bd23daee73977b..d261e4a4f338a56 100644
--- a/Doc/library/__future__.rst
+++ b/Doc/library/__future__.rst
@@ -22,42 +22,48 @@
can be inspected programmatically via importing :mod:`__future__` and examining
its contents.
-Each statement in :file:`__future__.py` is of the form::
+.. _future-classes:
- FeatureName = _Feature(OptionalRelease, MandatoryRelease,
- CompilerFlag)
+.. class:: _Feature
+ Each statement in :file:`__future__.py` is of the form::
-where, normally, *OptionalRelease* is less than *MandatoryRelease*, and both are
-5-tuples of the same form as :data:`sys.version_info`::
+ FeatureName = _Feature(OptionalRelease, MandatoryRelease,
+ CompilerFlag)
- (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int
- PY_MINOR_VERSION, # the 1; an int
- PY_MICRO_VERSION, # the 0; an int
- PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string
- PY_RELEASE_SERIAL # the 3; an int
- )
+ where, normally, *OptionalRelease* is less than *MandatoryRelease*, and both are
+ 5-tuples of the same form as :data:`sys.version_info`::
-*OptionalRelease* records the first release in which the feature was accepted.
+ (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int
+ PY_MINOR_VERSION, # the 1; an int
+ PY_MICRO_VERSION, # the 0; an int
+ PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string
+ PY_RELEASE_SERIAL # the 3; an int
+ )
-In the case of a *MandatoryRelease* that has not yet occurred,
-*MandatoryRelease* predicts the release in which the feature will become part of
-the language.
+.. method:: _Feature.getOptionalRelease()
-Else *MandatoryRelease* records when the feature became part of the language; in
-releases at or after that, modules no longer need a future statement to use the
-feature in question, but may continue to use such imports.
+ *OptionalRelease* records the first release in which the feature was accepted.
-*MandatoryRelease* may also be ``None``, meaning that a planned feature got
-dropped.
+.. method:: _Feature.getMandatoryRelease()
-Instances of class :class:`_Feature` have two corresponding methods,
-:meth:`getOptionalRelease` and :meth:`getMandatoryRelease`.
+ In the case of a *MandatoryRelease* that has not yet occurred,
+ *MandatoryRelease* predicts the release in which the feature will become part of
+ the language.
-*CompilerFlag* is the (bitfield) flag that should be passed in the fourth
-argument to the built-in function :func:`compile` to enable the feature in
-dynamically compiled code. This flag is stored in the :attr:`compiler_flag`
-attribute on :class:`_Feature` instances.
+ Else *MandatoryRelease* records when the feature became part of the language; in
+ releases at or after that, modules no longer need a future statement to use the
+ feature in question, but may continue to use such imports.
+
+ *MandatoryRelease* may also be ``None``, meaning that a planned feature got
+ dropped or that it is not yet decided.
+
+.. attribute:: _Feature.compiler_flag
+
+ *CompilerFlag* is the (bitfield) flag that should be passed in the fourth
+ argument to the built-in function :func:`compile` to enable the feature in
+ dynamically compiled code. This flag is stored in the :attr:`_Feature.compiler_flag`
+ attribute on :class:`_Feature` instances.
No feature description will ever be deleted from :mod:`__future__`. Since its
introduction in Python 2.1 the following features have found their way into the
diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst
index d0a65e76b842375..c999253f781b107 100644
--- a/Doc/library/__main__.rst
+++ b/Doc/library/__main__.rst
@@ -54,45 +54,45 @@ The top-level code environment can be:
* the scope of an interactive prompt::
- >>> __name__
- '__main__'
+ >>> __name__
+ '__main__'
* the Python module passed to the Python interpreter as a file argument:
- .. code-block:: shell-session
+ .. code-block:: shell-session
- $ python3 helloworld.py
- Hello, world!
+ $ python helloworld.py
+ Hello, world!
* the Python module or package passed to the Python interpreter with the
:option:`-m` argument:
- .. code-block:: shell-session
+ .. code-block:: shell-session
- $ python3 -m tarfile
- usage: tarfile.py [-h] [-v] (...)
+ $ python -m tarfile
+ usage: tarfile.py [-h] [-v] (...)
* Python code read by the Python interpreter from standard input:
- .. code-block:: shell-session
+ .. code-block:: shell-session
- $ echo "import this" | python3
- The Zen of Python, by Tim Peters
+ $ echo "import this" | python
+ The Zen of Python, by Tim Peters
- Beautiful is better than ugly.
- Explicit is better than implicit.
- ...
+ Beautiful is better than ugly.
+ Explicit is better than implicit.
+ ...
* Python code passed to the Python interpreter with the :option:`-c` argument:
- .. code-block:: shell-session
+ .. code-block:: shell-session
- $ python3 -c "import this"
- The Zen of Python, by Tim Peters
+ $ python -c "import this"
+ The Zen of Python, by Tim Peters
- Beautiful is better than ugly.
- Explicit is better than implicit.
- ...
+ Beautiful is better than ugly.
+ Explicit is better than implicit.
+ ...
In each of these situations, the top-level module's ``__name__`` is set to
``'__main__'``.
@@ -102,9 +102,9 @@ top-level environment by checking its own ``__name__``, which allows a common
idiom for conditionally executing code when the module is not initialized from
an import statement::
- if __name__ == '__main__':
- # Execute when the module is not initialized from an import statement.
- ...
+ if __name__ == '__main__':
+ # Execute when the module is not initialized from an import statement.
+ ...
.. seealso::
@@ -124,7 +124,7 @@ This is where using the ``if __name__ == '__main__'`` code block comes in
handy. Code within this block won't run unless the module is executed in the
top-level environment.
-Putting as few statements as possible in the block below ``if __name___ ==
+Putting as few statements as possible in the block below ``if __name__ ==
'__main__'`` can improve code clarity and correctness. Most often, a function
named ``main`` encapsulates the program's primary behavior::
@@ -178,7 +178,7 @@ that your function will return some value acceptable as an input to
returned if your function does not have a return statement).
By proactively following this convention ourselves, our module will have the
-same behavior when run directly (i.e. ``python3 echo.py``) as it will have if
+same behavior when run directly (i.e. ``python echo.py``) as it will have if
we later package it as a console script entry-point in a pip-installable
package.
@@ -215,7 +215,7 @@ directly from the command line using the :option:`-m` flag. For example:
.. code-block:: shell-session
- $ python3 -m bandclass
+ $ python -m bandclass
This command will cause ``__main__.py`` to run. How you utilize this mechanism
will depend on the nature of the package you are writing, but in this
@@ -227,7 +227,7 @@ students::
import sys
from .student import search_students
- student_name = sys.argv[2] if len(sys.argv) >= 2 else ''
+ student_name = sys.argv[1] if len(sys.argv) >= 2 else ''
print(f'Found student: {search_students(student_name)}')
Note that ``from .student import search_students`` is an example of a relative
@@ -238,9 +238,9 @@ package. For more details, see :ref:`intra-package-references` in the
Idiomatic Usage
^^^^^^^^^^^^^^^
-The contents of ``__main__.py`` typically isn't fenced with
-``if __name__ == '__main__'`` blocks. Instead, those files are kept short,
-functions to execute from other modules. Those other modules can then be
+The content of ``__main__.py`` typically isn't fenced with an
+``if __name__ == '__main__'`` block. Instead, those files are kept
+short and import functions to execute from other modules. Those other modules can then be
easily unit-tested and are properly reusable.
If used, an ``if __name__ == '__main__'`` block will still work as expected
@@ -259,7 +259,7 @@ one mentioned below are preferred.
See :mod:`venv` for an example of a package with a minimal ``__main__.py``
in the standard library. It doesn't contain a ``if __name__ == '__main__'``
- block. You can invoke it with ``python3 -m venv [directory]``.
+ block. You can invoke it with ``python -m venv [directory]``.
See :mod:`runpy` for more details on the :option:`-m` flag to the
interpreter executable.
@@ -320,7 +320,7 @@ Now, if we started our program, the result would look like this:
.. code-block:: shell-session
- $ python3 start.py
+ $ python start.py
Define the variable `my_name`!
The exit code of the program would be 1, indicating an error. Uncommenting the
@@ -329,19 +329,19 @@ status code 0, indicating success:
.. code-block:: shell-session
- $ python3 start.py
+ $ python start.py
Dinsdale found in file /path/to/start.py
Note that importing ``__main__`` doesn't cause any issues with unintentionally
running top-level code meant for script use which is put in the
``if __name__ == "__main__"`` block of the ``start`` module. Why does this work?
-Python inserts an empty ``__main__`` module in :attr:`sys.modules` at
+Python inserts an empty ``__main__`` module in :data:`sys.modules` at
interpreter startup, and populates it by running top-level code. In our example
this is the ``start`` module which runs line by line and imports ``namely``.
In turn, ``namely`` imports ``__main__`` (which is really ``start``). That's an
import cycle! Fortunately, since the partially populated ``__main__``
-module is present in :attr:`sys.modules`, Python passes that to ``namely``.
+module is present in :data:`sys.modules`, Python passes that to ``namely``.
See :ref:`Special considerations for __main__ ` in the
import system's reference for details on how this works.
diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst
index 9df9e7914e093b1..297f50a46e06928 100644
--- a/Doc/library/_thread.rst
+++ b/Doc/library/_thread.rst
@@ -57,6 +57,8 @@ This module defines the following constants and functions:
When the function raises a :exc:`SystemExit` exception, it is silently
ignored.
+ .. audit-event:: _thread.start_new_thread function,args,kwargs start_new_thread
+
.. versionchanged:: 3.8
:func:`sys.unraisablehook` is now used to handle unhandled exceptions.
@@ -68,10 +70,10 @@ This module defines the following constants and functions:
there is no guarantee that the interruption will happen immediately.
If given, *signum* is the number of the signal to simulate.
- If *signum* is not given, :data:`signal.SIGINT` is simulated.
+ If *signum* is not given, :const:`signal.SIGINT` is simulated.
If the given signal isn't handled by Python (it was set to
- :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does
+ :const:`signal.SIG_DFL` or :const:`signal.SIG_IGN`), this function does
nothing.
.. versionchanged:: 3.10
@@ -118,10 +120,13 @@ This module defines the following constants and functions:
Its value may be used to uniquely identify this particular thread system-wide
(until the thread terminates, after which the value may be recycled by the OS).
- .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD.
+ .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD.
.. versionadded:: 3.8
+ .. versionchanged:: 3.13
+ Added support for GNU/kFreeBSD.
+
.. function:: stack_size([size])
@@ -148,8 +153,8 @@ This module defines the following constants and functions:
.. data:: TIMEOUT_MAX
The maximum value allowed for the *timeout* parameter of
- :meth:`Lock.acquire`. Specifying a timeout greater than this value will
- raise an :exc:`OverflowError`.
+ :meth:`Lock.acquire `. Specifying a timeout greater
+ than this value will raise an :exc:`OverflowError`.
.. versionadded:: 3.2
@@ -206,7 +211,7 @@ In addition to these methods, lock objects can also be used via the
**Caveats:**
- .. index:: module: signal
+.. index:: pair: module; signal
* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt`
exception will be received by an arbitrary thread. (When the :mod:`signal`
@@ -215,8 +220,9 @@ In addition to these methods, lock objects can also be used via the
* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is
equivalent to calling :func:`_thread.exit`.
-* It is not possible to interrupt the :meth:`acquire` method on a lock --- the
- :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired.
+* It is not possible to interrupt the :meth:`~threading.Lock.acquire` method on
+ a lock --- the :exc:`KeyboardInterrupt` exception will happen after the lock
+ has been acquired.
* When the main thread exits, it is system defined whether the other threads
survive. On most systems, they are killed without executing
diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst
index 3b74622e7ff46c8..fb4f9da169c5abf 100644
--- a/Doc/library/abc.rst
+++ b/Doc/library/abc.rst
@@ -21,7 +21,7 @@ The :mod:`collections` module has some concrete classes that derive from
ABCs; these can, of course, be further derived. In addition, the
:mod:`collections.abc` submodule has some ABCs that can be used to test whether
a class or instance provides a particular interface, for example, if it is
-hashable or if it is a mapping.
+:term:`hashable` or if it is a mapping.
This module provides the metaclass :class:`ABCMeta` for defining ABCs and
@@ -154,7 +154,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance:
Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``,
even though it does not define an :meth:`~iterator.__iter__` method (it uses
the old-style iterable protocol, defined in terms of :meth:`__len__` and
- :meth:`__getitem__`). Note that this will not make ``get_iterator``
+ :meth:`~object.__getitem__`). Note that this will not make ``get_iterator``
available as a method of ``Foo``, so it is provided separately.
diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst
deleted file mode 100644
index 9f20a30193fa70d..000000000000000
--- a/Doc/library/aifc.rst
+++ /dev/null
@@ -1,247 +0,0 @@
-:mod:`aifc` --- Read and write AIFF and AIFC files
-==================================================
-
-.. module:: aifc
- :synopsis: Read and write audio files in AIFF or AIFC format.
- :deprecated:
-
-**Source code:** :source:`Lib/aifc.py`
-
-.. index::
- single: Audio Interchange File Format
- single: AIFF
- single: AIFF-C
-
-
-.. deprecated-removed:: 3.11 3.13
- The :mod:`aifc` module is deprecated
- (see :pep:`PEP 594 <594#aifc>` for details).
-
---------------
-
-This module provides support for reading and writing AIFF and AIFF-C files.
-AIFF is Audio Interchange File Format, a format for storing digital audio
-samples in a file. AIFF-C is a newer version of the format that includes the
-ability to compress the audio data.
-
-Audio files have a number of parameters that describe the audio data. The
-sampling rate or frame rate is the number of times per second the sound is
-sampled. The number of channels indicate if the audio is mono, stereo, or
-quadro. Each frame consists of one sample per channel. The sample size is the
-size in bytes of each sample. Thus a frame consists of
-``nchannels * samplesize`` bytes, and a second's worth of audio consists of
-``nchannels * samplesize * framerate`` bytes.
-
-For example, CD quality audio has a sample size of two bytes (16 bits), uses two
-channels (stereo) and has a frame rate of 44,100 frames/second. This gives a
-frame size of 4 bytes (2\*2), and a second's worth occupies 2\*2\*44100 bytes
-(176,400 bytes).
-
-Module :mod:`aifc` defines the following function:
-
-
-.. function:: open(file, mode=None)
-
- Open an AIFF or AIFF-C file and return an object instance with methods that are
- described below. The argument *file* is either a string naming a file or a
- :term:`file object`. *mode* must be ``'r'`` or ``'rb'`` when the file must be
- opened for reading, or ``'w'`` or ``'wb'`` when the file must be opened for writing.
- If omitted, ``file.mode`` is used if it exists, otherwise ``'rb'`` is used. When
- used for writing, the file object should be seekable, unless you know ahead of
- time how many samples you are going to write in total and use
- :meth:`writeframesraw` and :meth:`setnframes`.
- The :func:`.open` function may be used in a :keyword:`with` statement. When
- the :keyword:`!with` block completes, the :meth:`~aifc.close` method is called.
-
- .. versionchanged:: 3.4
- Support for the :keyword:`with` statement was added.
-
-Objects returned by :func:`.open` when a file is opened for reading have the
-following methods:
-
-
-.. method:: aifc.getnchannels()
-
- Return the number of audio channels (1 for mono, 2 for stereo).
-
-
-.. method:: aifc.getsampwidth()
-
- Return the size in bytes of individual samples.
-
-
-.. method:: aifc.getframerate()
-
- Return the sampling rate (number of audio frames per second).
-
-
-.. method:: aifc.getnframes()
-
- Return the number of audio frames in the file.
-
-
-.. method:: aifc.getcomptype()
-
- Return a bytes array of length 4 describing the type of compression
- used in the audio file. For AIFF files, the returned value is
- ``b'NONE'``.
-
-
-.. method:: aifc.getcompname()
-
- Return a bytes array convertible to a human-readable description
- of the type of compression used in the audio file. For AIFF files,
- the returned value is ``b'not compressed'``.
-
-
-.. method:: aifc.getparams()
-
- Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth,
- framerate, nframes, comptype, compname)``, equivalent to output of the
- :meth:`get\*` methods.
-
-
-.. method:: aifc.getmarkers()
-
- Return a list of markers in the audio file. A marker consists of a tuple of
- three elements. The first is the mark ID (an integer), the second is the mark
- position in frames from the beginning of the data (an integer), the third is the
- name of the mark (a string).
-
-
-.. method:: aifc.getmark(id)
-
- Return the tuple as described in :meth:`getmarkers` for the mark with the given
- *id*.
-
-
-.. method:: aifc.readframes(nframes)
-
- Read and return the next *nframes* frames from the audio file. The returned
- data is a string containing for each frame the uncompressed samples of all
- channels.
-
-
-.. method:: aifc.rewind()
-
- Rewind the read pointer. The next :meth:`readframes` will start from the
- beginning.
-
-
-.. method:: aifc.setpos(pos)
-
- Seek to the specified frame number.
-
-
-.. method:: aifc.tell()
-
- Return the current frame number.
-
-
-.. method:: aifc.close()
-
- Close the AIFF file. After calling this method, the object can no longer be
- used.
-
-Objects returned by :func:`.open` when a file is opened for writing have all the
-above methods, except for :meth:`readframes` and :meth:`setpos`. In addition
-the following methods exist. The :meth:`get\*` methods can only be called after
-the corresponding :meth:`set\*` methods have been called. Before the first
-:meth:`writeframes` or :meth:`writeframesraw`, all parameters except for the
-number of frames must be filled in.
-
-
-.. method:: aifc.aiff()
-
- Create an AIFF file. The default is that an AIFF-C file is created, unless the
- name of the file ends in ``'.aiff'`` in which case the default is an AIFF file.
-
-
-.. method:: aifc.aifc()
-
- Create an AIFF-C file. The default is that an AIFF-C file is created, unless
- the name of the file ends in ``'.aiff'`` in which case the default is an AIFF
- file.
-
-
-.. method:: aifc.setnchannels(nchannels)
-
- Specify the number of channels in the audio file.
-
-
-.. method:: aifc.setsampwidth(width)
-
- Specify the size in bytes of audio samples.
-
-
-.. method:: aifc.setframerate(rate)
-
- Specify the sampling frequency in frames per second.
-
-
-.. method:: aifc.setnframes(nframes)
-
- Specify the number of frames that are to be written to the audio file. If this
- parameter is not set, or not set correctly, the file needs to support seeking.
-
-
-.. method:: aifc.setcomptype(type, name)
-
- .. index::
- single: u-LAW
- single: A-LAW
- single: G.722
-
- Specify the compression type. If not specified, the audio data will
- not be compressed. In AIFF files, compression is not possible.
- The name parameter should be a human-readable description of the
- compression type as a bytes array, the type parameter should be a
- bytes array of length 4. Currently the following compression types
- are supported: ``b'NONE'``, ``b'ULAW'``, ``b'ALAW'``, ``b'G722'``.
-
-
-.. method:: aifc.setparams(nchannels, sampwidth, framerate, comptype, compname)
-
- Set all the above parameters at once. The argument is a tuple consisting of the
- various parameters. This means that it is possible to use the result of a
- :meth:`getparams` call as argument to :meth:`setparams`.
-
-
-.. method:: aifc.setmark(id, pos, name)
-
- Add a mark with the given id (larger than 0), and the given name at the given
- position. This method can be called at any time before :meth:`close`.
-
-
-.. method:: aifc.tell()
- :noindex:
-
- Return the current write position in the output file. Useful in combination
- with :meth:`setmark`.
-
-
-.. method:: aifc.writeframes(data)
-
- Write data to the output file. This method can only be called after the audio
- file parameters have been set.
-
- .. versionchanged:: 3.4
- Any :term:`bytes-like object` is now accepted.
-
-
-.. method:: aifc.writeframesraw(data)
-
- Like :meth:`writeframes`, except that the header of the audio file is not
- updated.
-
- .. versionchanged:: 3.4
- Any :term:`bytes-like object` is now accepted.
-
-
-.. method:: aifc.close()
- :noindex:
-
- Close the AIFF file. The header of the file is updated to reflect the actual
- size of the audio data. After calling this method, the object can no longer be
- used.
-
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index c55d94421e5b14e..fbffa71d2007359 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -31,12 +31,12 @@ Core Functionality
The :mod:`argparse` module's support for command-line interfaces is built
around an instance of :class:`argparse.ArgumentParser`. It is a container for
-argument specifications and has options that apply the parser as whole::
+argument specifications and has options that apply to the parser as whole::
parser = argparse.ArgumentParser(
- prog = 'ProgramName',
- description = 'What the program does',
- epilog = 'Text at the bottom of help')
+ prog='ProgramName',
+ description='What the program does',
+ epilog='Text at the bottom of help')
The :meth:`ArgumentParser.add_argument` method attaches individual argument
specifications to the parser. It supports positional arguments, options that
@@ -57,20 +57,20 @@ the extracted data in a :class:`argparse.Namespace` object::
Quick Links for add_argument()
------------------------------
-====================== =========================================================== ==========================================================================================================================
-Name Description Values
-====================== =========================================================== ==========================================================================================================================
-action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'``
-choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance
-const_ Store a constant value
-default_ Default value used when an argument is not provided Defaults to ``None``
-dest_ Specify the attribute name used in the result namespace
-help_ Help message for an argument
-metavar_ Alternate display name for the argument as shown in help
-nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, ``'+'``, or ``argparse.REMAINDER``
-required_ Indicate whether an argument is required or optional ``True`` or ``False``
-type_ Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function
-====================== =========================================================== ==========================================================================================================================
+============================ =========================================================== ==========================================================================================================================
+Name Description Values
+============================ =========================================================== ==========================================================================================================================
+action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'``
+choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance
+const_ Store a constant value
+default_ Default value used when an argument is not provided Defaults to ``None``
+dest_ Specify the attribute name used in the result namespace
+help_ Help message for an argument
+metavar_ Alternate display name for the argument as shown in help
+nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, or ``'+'``
+required_ Indicate whether an argument is required or optional ``True`` or ``False``
+:ref:`type ` Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function
+============================ =========================================================== ==========================================================================================================================
Example
@@ -565,6 +565,7 @@ arguments they contain. For example::
>>> with open('args.txt', 'w', encoding=sys.getfilesystemencoding()) as fp:
... fp.write('-f\nbar')
+ ...
>>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@')
>>> parser.add_argument('-f')
>>> parser.parse_args(['-f', 'foo', '@args.txt'])
@@ -584,7 +585,7 @@ arguments will never be treated as file references.
.. versionchanged:: 3.12
:class:`ArgumentParser` changed encoding and errors to read arguments files
- from default (e.g. :func:`locale.getpreferredencoding(False)` and
+ from default (e.g. :func:`locale.getpreferredencoding(False) ` and
``"strict"``) to :term:`filesystem encoding and error handler`.
Arguments file should be encoded in UTF-8 instead of ANSI Codepage on Windows.
@@ -764,7 +765,7 @@ The add_argument() method
* type_ - The type to which the command-line argument should be converted.
- * choices_ - A container of the allowable values for the argument.
+ * choices_ - A sequence of the allowable values for the argument.
* required_ - Whether or not the command-line option may be omitted
(optionals only).
@@ -1131,7 +1132,7 @@ command-line argument was not present::
Namespace(foo='1')
-.. _type:
+.. _argparse-type:
type
^^^^
@@ -1190,7 +1191,7 @@ done downstream after the arguments are parsed.
For example, JSON or YAML conversions have complex error cases that require
better reporting than can be given by the ``type`` keyword. A
:exc:`~json.JSONDecodeError` would not be well formatted and a
-:exc:`FileNotFound` exception would not be handled at all.
+:exc:`FileNotFoundError` exception would not be handled at all.
Even :class:`~argparse.FileType` has its limitations for use with the ``type``
keyword. If one argument uses *FileType* and then a subsequent argument fails,
@@ -1208,7 +1209,7 @@ choices
^^^^^^^
Some command-line arguments should be selected from a restricted set of values.
-These can be handled by passing a container object as the *choices* keyword
+These can be handled by passing a sequence object as the *choices* keyword
argument to :meth:`~ArgumentParser.add_argument`. When the command line is
parsed, argument values will be checked, and an error message will be displayed
if the argument was not one of the acceptable values::
@@ -1222,9 +1223,9 @@ if the argument was not one of the acceptable values::
game.py: error: argument move: invalid choice: 'fire' (choose from 'rock',
'paper', 'scissors')
-Note that inclusion in the *choices* container is checked after any type_
+Note that inclusion in the *choices* sequence is checked after any type_
conversions have been performed, so the type of the objects in the *choices*
-container should match the type_ specified::
+sequence should match the type_ specified::
>>> parser = argparse.ArgumentParser(prog='doors.py')
>>> parser.add_argument('door', type=int, choices=range(1, 4))
@@ -1234,8 +1235,8 @@ container should match the type_ specified::
usage: doors.py [-h] {1,2,3}
doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3)
-Any container can be passed as the *choices* value, so :class:`list` objects,
-:class:`set` objects, and custom containers are all supported.
+Any sequence can be passed as the *choices* value, so :class:`list` objects,
+:class:`tuple` objects, and custom sequences are all supported.
Use of :class:`enum.Enum` is not recommended because it is difficult to
control its appearance in usage, help, and error messages.
@@ -1444,7 +1445,7 @@ Action classes
Action classes implement the Action API, a callable which returns a callable
which processes arguments from the command-line. Any object which follows
this API may be passed as the ``action`` parameter to
-:meth:`add_argument`.
+:meth:`~ArgumentParser.add_argument`.
.. class:: Action(option_strings, dest, nargs=None, const=None, default=None, \
type=None, choices=None, required=False, help=None, \
@@ -1722,7 +1723,7 @@ Sub-commands
:class:`ArgumentParser` supports the creation of such sub-commands with the
:meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally
called with no arguments and returns a special action object. This object
- has a single method, :meth:`~ArgumentParser.add_parser`, which takes a
+ has a single method, :meth:`~_SubParsersAction.add_parser`, which takes a
command name and any :class:`ArgumentParser` constructor arguments, and
returns an :class:`ArgumentParser` object that can be modified as usual.
@@ -1788,7 +1789,7 @@ Sub-commands
for that particular parser will be printed. The help message will not
include parent parser or sibling parser messages. (A help message for each
subparser command, however, can be given by supplying the ``help=`` argument
- to :meth:`add_parser` as above.)
+ to :meth:`~_SubParsersAction.add_parser` as above.)
::
@@ -1866,7 +1867,7 @@ Sub-commands
...
>>> # create the top-level parser
>>> parser = argparse.ArgumentParser()
- >>> subparsers = parser.add_subparsers()
+ >>> subparsers = parser.add_subparsers(required=True)
>>>
>>> # create the parser for the "foo" command
>>> parser_foo = subparsers.add_parser('foo')
@@ -1945,7 +1946,7 @@ Argument groups
.. method:: ArgumentParser.add_argument_group(title=None, description=None)
By default, :class:`ArgumentParser` groups command-line arguments into
- "positional arguments" and "optional arguments" when displaying help
+ "positional arguments" and "options" when displaying help
messages. When there is a better conceptual grouping of arguments than this
default one, appropriate groups can be created using the
:meth:`add_argument_group` method::
@@ -2156,7 +2157,7 @@ the populated namespace and the list of remaining argument strings.
.. warning::
:ref:`Prefix matching ` rules apply to
- :meth:`parse_known_args`. The parser may consume an option even if it's just
+ :meth:`~ArgumentParser.parse_known_args`. The parser may consume an option even if it's just
a prefix of one of its known options, instead of leaving it in the remaining
arguments list.
@@ -2217,7 +2218,7 @@ support this parsing style.
These parsers do not support all the argparse features, and will raise
exceptions if unsupported features are used. In particular, subparsers,
-``argparse.REMAINDER``, and mutually exclusive groups that include both
+and mutually exclusive groups that include both
optionals and positionals are not supported.
The following example shows the difference between
@@ -2294,3 +2295,17 @@ A partial upgrade path from :mod:`optparse` to :mod:`argparse`:
* Replace the OptionParser constructor ``version`` argument with a call to
``parser.add_argument('--version', action='version', version='')``.
+
+Exceptions
+----------
+
+.. exception:: ArgumentError
+
+ An error from creating or using an argument (optional or positional).
+
+ The string value of this exception is the message, augmented with
+ information about the argument that caused it.
+
+.. exception:: ArgumentTypeError
+
+ Raised when something goes wrong converting a command line string to a type.
diff --git a/Doc/library/array.rst b/Doc/library/array.rst
index 95f1eaf401b0526..ad622627724217a 100644
--- a/Doc/library/array.rst
+++ b/Doc/library/array.rst
@@ -24,6 +24,8 @@ defined:
+-----------+--------------------+-------------------+-----------------------+-------+
| ``'u'`` | wchar_t | Unicode character | 2 | \(1) |
+-----------+--------------------+-------------------+-----------------------+-------+
+| ``'w'`` | Py_UCS4 | Unicode character | 4 | |
++-----------+--------------------+-------------------+-----------------------+-------+
| ``'h'`` | signed short | int | 2 | |
+-----------+--------------------+-------------------+-----------------------+-------+
| ``'H'`` | unsigned short | int | 2 | |
@@ -51,16 +53,17 @@ Notes:
It can be 16 bits or 32 bits depending on the platform.
.. versionchanged:: 3.9
- ``array('u')`` now uses ``wchar_t`` as C type instead of deprecated
+ ``array('u')`` now uses :c:type:`wchar_t` as C type instead of deprecated
``Py_UNICODE``. This change doesn't affect its behavior because
- ``Py_UNICODE`` is alias of ``wchar_t`` since Python 3.3.
+ ``Py_UNICODE`` is alias of :c:type:`wchar_t` since Python 3.3.
- .. deprecated-removed:: 3.3 4.0
+ .. deprecated-removed:: 3.3 3.16
+ Please migrate to ``'w'`` typecode.
The actual representation of values is determined by the machine architecture
(strictly speaking, by the C implementation). The actual size can be accessed
-through the :attr:`itemsize` attribute.
+through the :attr:`array.itemsize` attribute.
The module defines the following item:
@@ -85,173 +88,173 @@ The module defines the following type:
to add initial items to the array. Otherwise, the iterable initializer is
passed to the :meth:`extend` method.
- .. audit-event:: array.__new__ typecode,initializer array.array
+ Array objects support the ordinary sequence operations of indexing, slicing,
+ concatenation, and multiplication. When using slice assignment, the assigned
+ value must be an array object with the same type code; in all other cases,
+ :exc:`TypeError` is raised. Array objects also implement the buffer interface,
+ and may be used wherever :term:`bytes-like objects ` are supported.
+ .. audit-event:: array.__new__ typecode,initializer array.array
-Array objects support the ordinary sequence operations of indexing, slicing,
-concatenation, and multiplication. When using slice assignment, the assigned
-value must be an array object with the same type code; in all other cases,
-:exc:`TypeError` is raised. Array objects also implement the buffer interface,
-and may be used wherever :term:`bytes-like objects ` are supported.
-The following data items and methods are also supported:
+ .. attribute:: typecode
-.. attribute:: array.typecode
+ The typecode character used to create the array.
- The typecode character used to create the array.
+ .. attribute:: itemsize
-.. attribute:: array.itemsize
+ The length in bytes of one array item in the internal representation.
- The length in bytes of one array item in the internal representation.
+ .. method:: append(x)
-.. method:: array.append(x)
+ Append a new item with value *x* to the end of the array.
- Append a new item with value *x* to the end of the array.
+ .. method:: buffer_info()
-.. method:: array.buffer_info()
+ Return a tuple ``(address, length)`` giving the current memory address and the
+ length in elements of the buffer used to hold array's contents. The size of the
+ memory buffer in bytes can be computed as ``array.buffer_info()[1] *
+ array.itemsize``. This is occasionally useful when working with low-level (and
+ inherently unsafe) I/O interfaces that require memory addresses, such as certain
+ :c:func:`!ioctl` operations. The returned numbers are valid as long as the array
+ exists and no length-changing operations are applied to it.
- Return a tuple ``(address, length)`` giving the current memory address and the
- length in elements of the buffer used to hold array's contents. The size of the
- memory buffer in bytes can be computed as ``array.buffer_info()[1] *
- array.itemsize``. This is occasionally useful when working with low-level (and
- inherently unsafe) I/O interfaces that require memory addresses, such as certain
- :c:func:`ioctl` operations. The returned numbers are valid as long as the array
- exists and no length-changing operations are applied to it.
+ .. note::
- .. note::
+ When using array objects from code written in C or C++ (the only way to
+ effectively make use of this information), it makes more sense to use the buffer
+ interface supported by array objects. This method is maintained for backward
+ compatibility and should be avoided in new code. The buffer interface is
+ documented in :ref:`bufferobjects`.
- When using array objects from code written in C or C++ (the only way to
- effectively make use of this information), it makes more sense to use the buffer
- interface supported by array objects. This method is maintained for backward
- compatibility and should be avoided in new code. The buffer interface is
- documented in :ref:`bufferobjects`.
+ .. method:: byteswap()
-.. method:: array.byteswap()
+ "Byteswap" all items of the array. This is only supported for values which are
+ 1, 2, 4, or 8 bytes in size; for other types of values, :exc:`RuntimeError` is
+ raised. It is useful when reading data from a file written on a machine with a
+ different byte order.
- "Byteswap" all items of the array. This is only supported for values which are
- 1, 2, 4, or 8 bytes in size; for other types of values, :exc:`RuntimeError` is
- raised. It is useful when reading data from a file written on a machine with a
- different byte order.
+ .. method:: count(x)
-.. method:: array.count(x)
+ Return the number of occurrences of *x* in the array.
- Return the number of occurrences of *x* in the array.
+ .. method:: extend(iterable)
-.. method:: array.extend(iterable)
+ Append items from *iterable* to the end of the array. If *iterable* is another
+ array, it must have *exactly* the same type code; if not, :exc:`TypeError` will
+ be raised. If *iterable* is not an array, it must be iterable and its elements
+ must be the right type to be appended to the array.
- Append items from *iterable* to the end of the array. If *iterable* is another
- array, it must have *exactly* the same type code; if not, :exc:`TypeError` will
- be raised. If *iterable* is not an array, it must be iterable and its elements
- must be the right type to be appended to the array.
+ .. method:: frombytes(s)
-.. method:: array.frombytes(s)
+ Appends items from the string, interpreting the string as an array of machine
+ values (as if it had been read from a file using the :meth:`fromfile` method).
- Appends items from the string, interpreting the string as an array of machine
- values (as if it had been read from a file using the :meth:`fromfile` method).
+ .. versionadded:: 3.2
+ :meth:`!fromstring` is renamed to :meth:`frombytes` for clarity.
- .. versionadded:: 3.2
- :meth:`fromstring` is renamed to :meth:`frombytes` for clarity.
+ .. method:: fromfile(f, n)
-.. method:: array.fromfile(f, n)
+ Read *n* items (as machine values) from the :term:`file object` *f* and append
+ them to the end of the array. If less than *n* items are available,
+ :exc:`EOFError` is raised, but the items that were available are still
+ inserted into the array.
- Read *n* items (as machine values) from the :term:`file object` *f* and append
- them to the end of the array. If less than *n* items are available,
- :exc:`EOFError` is raised, but the items that were available are still
- inserted into the array.
+ .. method:: fromlist(list)
-.. method:: array.fromlist(list)
+ Append items from the list. This is equivalent to ``for x in list:
+ a.append(x)`` except that if there is a type error, the array is unchanged.
- Append items from the list. This is equivalent to ``for x in list:
- a.append(x)`` except that if there is a type error, the array is unchanged.
+ .. method:: fromunicode(s)
-.. method:: array.fromunicode(s)
+ Extends this array with data from the given unicode string.
+ The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised.
+ Use ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an
+ array of some other type.
- Extends this array with data from the given unicode string. The array must
- be a type ``'u'`` array; otherwise a :exc:`ValueError` is raised. Use
- ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an
- array of some other type.
+ .. method:: index(x[, start[, stop]])
-.. method:: array.index(x[, start[, stop]])
+ Return the smallest *i* such that *i* is the index of the first occurrence of
+ *x* in the array. The optional arguments *start* and *stop* can be
+ specified to search for *x* within a subsection of the array. Raise
+ :exc:`ValueError` if *x* is not found.
- Return the smallest *i* such that *i* is the index of the first occurrence of
- *x* in the array. The optional arguments *start* and *stop* can be
- specified to search for *x* within a subsection of the array. Raise
- :exc:`ValueError` if *x* is not found.
+ .. versionchanged:: 3.10
+ Added optional *start* and *stop* parameters.
- .. versionchanged:: 3.10
- Added optional *start* and *stop* parameters.
-.. method:: array.insert(i, x)
+ .. method:: insert(i, x)
- Insert a new item with value *x* in the array before position *i*. Negative
- values are treated as being relative to the end of the array.
+ Insert a new item with value *x* in the array before position *i*. Negative
+ values are treated as being relative to the end of the array.
-.. method:: array.pop([i])
+ .. method:: pop([i])
- Removes the item with the index *i* from the array and returns it. The optional
- argument defaults to ``-1``, so that by default the last item is removed and
- returned.
+ Removes the item with the index *i* from the array and returns it. The optional
+ argument defaults to ``-1``, so that by default the last item is removed and
+ returned.
-.. method:: array.remove(x)
+ .. method:: remove(x)
- Remove the first occurrence of *x* from the array.
+ Remove the first occurrence of *x* from the array.
-.. method:: array.reverse()
+ .. method:: reverse()
- Reverse the order of the items in the array.
+ Reverse the order of the items in the array.
-.. method:: array.tobytes()
+ .. method:: tobytes()
- Convert the array to an array of machine values and return the bytes
- representation (the same sequence of bytes that would be written to a file by
- the :meth:`tofile` method.)
+ Convert the array to an array of machine values and return the bytes
+ representation (the same sequence of bytes that would be written to a file by
+ the :meth:`tofile` method.)
- .. versionadded:: 3.2
- :meth:`tostring` is renamed to :meth:`tobytes` for clarity.
+ .. versionadded:: 3.2
+ :meth:`!tostring` is renamed to :meth:`tobytes` for clarity.
-.. method:: array.tofile(f)
+ .. method:: tofile(f)
- Write all items (as machine values) to the :term:`file object` *f*.
+ Write all items (as machine values) to the :term:`file object` *f*.
-.. method:: array.tolist()
+ .. method:: tolist()
- Convert the array to an ordinary list with the same items.
+ Convert the array to an ordinary list with the same items.
-.. method:: array.tounicode()
+ .. method:: tounicode()
- Convert the array to a unicode string. The array must be a type ``'u'`` array;
- otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to
- obtain a unicode string from an array of some other type.
+ Convert the array to a unicode string. The array must have a type ``'u'`` or ``'w'``;
+ otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to
+ obtain a unicode string from an array of some other type.
When an array object is printed or converted to a string, it is represented as
``array(typecode, initializer)``. The *initializer* is omitted if the array is
-empty, otherwise it is a string if the *typecode* is ``'u'``, otherwise it is a
-list of numbers. The string is guaranteed to be able to be converted back to an
+empty, otherwise it is a string if the *typecode* is ``'u'`` or ``'w'``,
+otherwise it is a list of numbers.
+The string is guaranteed to be able to be converted back to an
array with the same type and value using :func:`eval`, so long as the
:class:`~array.array` class has been imported using ``from array import array``.
Examples::
array('l')
- array('u', 'hello \u2641')
+ array('w', 'hello \u2641')
array('l', [1, 2, 3, 4, 5])
array('d', [1.0, 2.0, 3.14])
@@ -261,10 +264,6 @@ Examples::
Module :mod:`struct`
Packing and unpacking of heterogeneous binary data.
- Module :mod:`xdrlib`
- Packing and unpacking of External Data Representation (XDR) data as used in some
- remote procedure call systems.
-
`NumPy `_
The NumPy package defines another array type.
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index 0811b3fa0e7842c..4ebbe0e5471c88c 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -146,6 +146,102 @@ Node classes
Snakes `__ project and
all its contributors.
+
+.. _ast-root-nodes:
+
+Root nodes
+^^^^^^^^^^
+
+.. class:: Module(body, type_ignores)
+
+ A Python module, as with :ref:`file input `.
+ Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*.
+
+ *body* is a :class:`list` of the module's :ref:`ast-statements`.
+
+ *type_ignores* is a :class:`list` of the module's type ignore comments;
+ see :func:`ast.parse` for more details.
+
+ .. doctest::
+
+ >>> print(ast.dump(ast.parse('x = 1'), indent=4))
+ Module(
+ body=[
+ Assign(
+ targets=[
+ Name(id='x', ctx=Store())],
+ value=Constant(value=1))],
+ type_ignores=[])
+
+
+.. class:: Expression(body)
+
+ A single Python :ref:`expression input `.
+ Node type generated by :func:`ast.parse` when *mode* is ``"eval"``.
+
+ *body* is a single node,
+ one of the :ref:`expression types `.
+
+ .. doctest::
+
+ >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4))
+ Expression(
+ body=Constant(value=123))
+
+
+.. class:: Interactive(body)
+
+ A single :ref:`interactive input `, like in :ref:`tut-interac`.
+ Node type generated by :func:`ast.parse` when *mode* is ``"single"``.
+
+ *body* is a :class:`list` of :ref:`statement nodes `.
+
+ .. doctest::
+
+ >>> print(ast.dump(ast.parse('x = 1; y = 2', mode='single'), indent=4))
+ Interactive(
+ body=[
+ Assign(
+ targets=[
+ Name(id='x', ctx=Store())],
+ value=Constant(value=1)),
+ Assign(
+ targets=[
+ Name(id='y', ctx=Store())],
+ value=Constant(value=2))])
+
+
+.. class:: FunctionType(argtypes, returns)
+
+ A representation of an old-style type comments for functions,
+ as Python versions prior to 3.5 didn't support :pep:`484` annotations.
+ Node type generated by :func:`ast.parse` when *mode* is ``"func_type"``.
+
+ Such type comments would look like this::
+
+ def sum_two_number(a, b):
+ # type: (int, int) -> int
+ return a + b
+
+ *argtypes* is a :class:`list` of :ref:`expression nodes `.
+
+ *returns* is a single :ref:`expression node `.
+
+ .. doctest::
+
+ >>> print(ast.dump(ast.parse('(int, str) -> List[int]', mode='func_type'), indent=4))
+ FunctionType(
+ argtypes=[
+ Name(id='int', ctx=Load()),
+ Name(id='str', ctx=Load())],
+ returns=Subscript(
+ value=Name(id='List', ctx=Load()),
+ slice=Name(id='int', ctx=Load()),
+ ctx=Load()))
+
+ .. versionadded:: 3.8
+
+
Literals
^^^^^^^^
@@ -344,6 +440,8 @@ Variables
type_ignores=[])
+.. _ast-expressions:
+
Expressions
^^^^^^^^^^^
@@ -481,17 +579,17 @@ Expressions
Comparison operator tokens.
-.. class:: Call(func, args, keywords, starargs, kwargs)
+.. class:: Call(func, args, keywords)
A function call. ``func`` is the function, which will often be a
:class:`Name` or :class:`Attribute` object. Of the arguments:
* ``args`` holds a list of the arguments passed by position.
- * ``keywords`` holds a list of :class:`keyword` objects representing
+ * ``keywords`` holds a list of :class:`.keyword` objects representing
arguments passed by keyword.
When creating a ``Call`` node, ``args`` and ``keywords`` are required, but
- they can be empty lists. ``starargs`` and ``kwargs`` are optional.
+ they can be empty lists.
.. doctest::
@@ -552,10 +650,10 @@ Expressions
.. class:: NamedExpr(target, value)
- A named expression. This AST node is produced by the assignment expressions
- operator (also known as the walrus operator). As opposed to the :class:`Assign`
- node in which the first argument can be multiple nodes, in this case both
- ``target`` and ``value`` must be single nodes.
+ A named expression. This AST node is produced by the assignment expressions
+ operator (also known as the walrus operator). As opposed to the :class:`Assign`
+ node in which the first argument can be multiple nodes, in this case both
+ ``target`` and ``value`` must be single nodes.
.. doctest::
@@ -565,6 +663,7 @@ Expressions
target=Name(id='x', ctx=Store()),
value=Constant(value=4)))
+ .. versionadded:: 3.8
Subscripting
~~~~~~~~~~~~
@@ -735,6 +834,9 @@ Comprehensions
ifs=[],
is_async=1)]))
+
+.. _ast-statements:
+
Statements
^^^^^^^^^^
@@ -917,6 +1019,26 @@ Statements
type_ignores=[])
+.. class:: TypeAlias(name, type_params, value)
+
+ A :ref:`type alias ` created through the :keyword:`type`
+ statement. ``name`` is the name of the alias, ``type_params`` is a list of
+ :ref:`type parameters `, and ``value`` is the value of the
+ type alias.
+
+ .. doctest::
+
+ >>> print(ast.dump(ast.parse('type Alias = int'), indent=4))
+ Module(
+ body=[
+ TypeAlias(
+ name=Name(id='Alias', ctx=Store()),
+ type_params=[],
+ value=Name(id='int', ctx=Load()))],
+ type_ignores=[])
+
+ .. versionadded:: 3.12
+
Other statements which are only applicable inside functions or loops are
described in other sections.
@@ -1198,6 +1320,7 @@ Control flow
finalbody=[])],
type_ignores=[])
+ .. versionadded:: 3.11
.. class:: ExceptHandler(type, name, body)
@@ -1287,6 +1410,8 @@ Pattern matching
that is being matched against the cases) and ``cases`` contains an iterable of
:class:`match_case` nodes with the different cases.
+ .. versionadded:: 3.10
+
.. class:: match_case(pattern, guard, body)
A single case pattern in a ``match`` statement. ``pattern`` contains the
@@ -1338,6 +1463,8 @@ Pattern matching
value=Constant(value=Ellipsis))])])],
type_ignores=[])
+ .. versionadded:: 3.10
+
.. class:: MatchValue(value)
A match literal or value pattern that compares by equality. ``value`` is
@@ -1365,6 +1492,8 @@ Pattern matching
value=Constant(value=Ellipsis))])])],
type_ignores=[])
+ .. versionadded:: 3.10
+
.. class:: MatchSingleton(value)
A match literal pattern that compares by identity. ``value`` is the
@@ -1390,6 +1519,8 @@ Pattern matching
value=Constant(value=Ellipsis))])])],
type_ignores=[])
+ .. versionadded:: 3.10
+
.. class:: MatchSequence(patterns)
A match sequence pattern. ``patterns`` contains the patterns to be matched
@@ -1421,6 +1552,8 @@ Pattern matching
value=Constant(value=Ellipsis))])])],
type_ignores=[])
+ .. versionadded:: 3.10
+
.. class:: MatchStar(name)
Matches the rest of the sequence in a variable length match sequence pattern.
@@ -1461,6 +1594,8 @@ Pattern matching
value=Constant(value=Ellipsis))])])],
type_ignores=[])
+ .. versionadded:: 3.10
+
.. class:: MatchMapping(keys, patterns, rest)
A match mapping pattern. ``keys`` is a sequence of expression nodes.
@@ -1507,6 +1642,8 @@ Pattern matching
value=Constant(value=Ellipsis))])])],
type_ignores=[])
+ .. versionadded:: 3.10
+
.. class:: MatchClass(cls, patterns, kwd_attrs, kwd_patterns)
A match class pattern. ``cls`` is an expression giving the nominal class to
@@ -1571,6 +1708,8 @@ Pattern matching
value=Constant(value=Ellipsis))])])],
type_ignores=[])
+ .. versionadded:: 3.10
+
.. class:: MatchAs(pattern, name)
A match "as-pattern", capture pattern or wildcard pattern. ``pattern``
@@ -1612,6 +1751,8 @@ Pattern matching
value=Constant(value=Ellipsis))])])],
type_ignores=[])
+ .. versionadded:: 3.10
+
.. class:: MatchOr(patterns)
A match "or-pattern". An or-pattern matches each of its subpatterns in turn
@@ -1644,11 +1785,96 @@ Pattern matching
value=Constant(value=Ellipsis))])])],
type_ignores=[])
+ .. versionadded:: 3.10
+
+.. _ast-type-params:
+
+Type parameters
+^^^^^^^^^^^^^^^
+
+:ref:`Type parameters ` can exist on classes, functions, and type
+aliases.
+
+.. class:: TypeVar(name, bound)
+
+ A :class:`typing.TypeVar`. ``name`` is the name of the type variable.
+ ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`,
+ it represents constraints; otherwise it represents the bound.
+
+ .. doctest::
+
+ >>> print(ast.dump(ast.parse("type Alias[T: int] = list[T]"), indent=4))
+ Module(
+ body=[
+ TypeAlias(
+ name=Name(id='Alias', ctx=Store()),
+ type_params=[
+ TypeVar(
+ name='T',
+ bound=Name(id='int', ctx=Load()))],
+ value=Subscript(
+ value=Name(id='list', ctx=Load()),
+ slice=Name(id='T', ctx=Load()),
+ ctx=Load()))],
+ type_ignores=[])
+
+ .. versionadded:: 3.12
+
+.. class:: ParamSpec(name)
+
+ A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification.
+
+ .. doctest::
+
+ >>> print(ast.dump(ast.parse("type Alias[**P] = Callable[P, int]"), indent=4))
+ Module(
+ body=[
+ TypeAlias(
+ name=Name(id='Alias', ctx=Store()),
+ type_params=[
+ ParamSpec(name='P')],
+ value=Subscript(
+ value=Name(id='Callable', ctx=Load()),
+ slice=Tuple(
+ elts=[
+ Name(id='P', ctx=Load()),
+ Name(id='int', ctx=Load())],
+ ctx=Load()),
+ ctx=Load()))],
+ type_ignores=[])
+
+ .. versionadded:: 3.12
+
+.. class:: TypeVarTuple(name)
+
+ A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple.
+
+ .. doctest::
+
+ >>> print(ast.dump(ast.parse("type Alias[*Ts] = tuple[*Ts]"), indent=4))
+ Module(
+ body=[
+ TypeAlias(
+ name=Name(id='Alias', ctx=Store()),
+ type_params=[
+ TypeVarTuple(name='Ts')],
+ value=Subscript(
+ value=Name(id='tuple', ctx=Load()),
+ slice=Tuple(
+ elts=[
+ Starred(
+ value=Name(id='Ts', ctx=Load()),
+ ctx=Load())],
+ ctx=Load()),
+ ctx=Load()))],
+ type_ignores=[])
+
+ .. versionadded:: 3.12
Function and class definitions
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment)
+.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment, type_params)
A function definition.
@@ -1658,11 +1884,15 @@ Function and class definitions
* ``decorator_list`` is the list of decorators to be applied, stored outermost
first (i.e. the first in the list will be applied last).
* ``returns`` is the return annotation.
+ * ``type_params`` is a list of :ref:`type parameters `.
.. attribute:: type_comment
``type_comment`` is an optional string with the type annotation as a comment.
+ .. versionchanged:: 3.12
+ Added ``type_params``.
+
.. class:: Lambda(args, body)
@@ -1748,7 +1978,8 @@ Function and class definitions
decorator_list=[
Name(id='decorator1', ctx=Load()),
Name(id='decorator2', ctx=Load())],
- returns=Constant(value='return annotation'))],
+ returns=Constant(value='return annotation'),
+ type_params=[])],
type_ignores=[])
@@ -1819,21 +2050,19 @@ Function and class definitions
type_ignores=[])
-.. class:: ClassDef(name, bases, keywords, starargs, kwargs, body, decorator_list)
+.. class:: ClassDef(name, bases, keywords, body, decorator_list, type_params)
A class definition.
* ``name`` is a raw string for the class name
* ``bases`` is a list of nodes for explicitly specified base classes.
- * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'.
+ * ``keywords`` is a list of :class:`.keyword` nodes, principally for 'metaclass'.
Other keywords will be passed to the metaclass, as per `PEP-3115
`_.
- * ``starargs`` and ``kwargs`` are each a single node, as in a function call.
- starargs will be expanded to join the list of base classes, and kwargs will
- be passed to the metaclass.
* ``body`` is a list of nodes representing the code within the class
definition.
* ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`.
+ * ``type_params`` is a list of :ref:`type parameters `.
.. doctest::
@@ -1858,17 +2087,24 @@ Function and class definitions
Pass()],
decorator_list=[
Name(id='decorator1', ctx=Load()),
- Name(id='decorator2', ctx=Load())])],
+ Name(id='decorator2', ctx=Load())],
+ type_params=[])],
type_ignores=[])
+ .. versionchanged:: 3.12
+ Added ``type_params``.
+
Async and await
^^^^^^^^^^^^^^^
-.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment)
+.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment, type_params)
An ``async def`` function definition. Has the same fields as
:class:`FunctionDef`.
+ .. versionchanged:: 3.12
+ Added ``type_params``.
+
.. class:: Await(value)
@@ -1898,7 +2134,8 @@ Async and await
func=Name(id='other_func', ctx=Load()),
args=[],
keywords=[])))],
- decorator_list=[])],
+ decorator_list=[],
+ type_params=[])],
type_ignores=[])
@@ -1923,10 +2160,12 @@ Async and await
Apart from the node classes, the :mod:`ast` module defines these utility functions
and classes for traversing abstract syntax trees:
-.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None)
+.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1)
Parse the source into an AST node. Equivalent to ``compile(source,
- filename, mode, ast.PyCF_ONLY_AST)``.
+ filename, mode, flags=FLAGS_VALUE, optimize=optimize)``,
+ where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0``
+ and ``ast.PyCF_OPTIMIZED_AST`` otherwise.
If ``type_comments=True`` is given, the parser is modified to check
and return type comments as specified by :pep:`484` and :pep:`526`.
@@ -1947,7 +2186,7 @@ and classes for traversing abstract syntax trees:
Currently ``major`` must equal to ``3``. For example, setting
``feature_version=(3, 4)`` will allow the use of ``async`` and
``await`` as variable names. The lowest supported version is
- ``(3, 4)``; the highest is ``sys.version_info[0:2]``.
+ ``(3, 7)``; the highest is ``sys.version_info[0:2]``.
If source contains a null character ('\0'), :exc:`ValueError` is raised.
@@ -1970,6 +2209,10 @@ and classes for traversing abstract syntax trees:
.. versionchanged:: 3.8
Added ``type_comments``, ``mode='func_type'`` and ``feature_version``.
+ .. versionchanged:: 3.13
+ The minimum supported version for feature_version is now (3,7)
+ The ``optimize`` argument was added.
+
.. function:: unparse(ast_obj)
@@ -2240,26 +2483,26 @@ The following options are accepted:
.. program:: ast
-.. cmdoption:: -h, --help
+.. option:: -h, --help
Show the help message and exit.
-.. cmdoption:: -m
- --mode
+.. option:: -m
+ --mode
Specify what kind of code must be compiled, like the *mode* argument
in :func:`parse`.
-.. cmdoption:: --no-type-comments
+.. option:: --no-type-comments
Don't parse type comments.
-.. cmdoption:: -a, --include-attributes
+.. option:: -a, --include-attributes
Include attributes such as line numbers and column offsets.
-.. cmdoption:: -i
- --indent
+.. option:: -i
+ --indent
Indentation of nodes in AST (number of spaces).
diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst
deleted file mode 100644
index 32e04ad6d195333..000000000000000
--- a/Doc/library/asynchat.rst
+++ /dev/null
@@ -1,217 +0,0 @@
-:mod:`asynchat` --- Asynchronous socket command/response handler
-================================================================
-
-.. module:: asynchat
- :synopsis: Support for asynchronous command/response protocols.
- :deprecated:
-
-.. moduleauthor:: Sam Rushing
-.. sectionauthor:: Steve Holden
-
-**Source code:** :source:`Lib/asynchat.py`
-
-.. deprecated-removed:: 3.6 3.12
- The :mod:`asynchat` module is deprecated
- (see :pep:`PEP 594 <594#asynchat>` for details).
- Please use :mod:`asyncio` instead.
-
---------------
-
-.. note::
-
- This module exists for backwards compatibility only. For new code we
- recommend using :mod:`asyncio`.
-
-This module builds on the :mod:`asyncore` infrastructure, simplifying
-asynchronous clients and servers and making it easier to handle protocols
-whose elements are terminated by arbitrary strings, or are of variable length.
-:mod:`asynchat` defines the abstract class :class:`async_chat` that you
-subclass, providing implementations of the :meth:`collect_incoming_data` and
-:meth:`found_terminator` methods. It uses the same asynchronous loop as
-:mod:`asyncore`, and the two types of channel, :class:`asyncore.dispatcher`
-and :class:`asynchat.async_chat`, can freely be mixed in the channel map.
-Typically an :class:`asyncore.dispatcher` server channel generates new
-:class:`asynchat.async_chat` channel objects as it receives incoming
-connection requests.
-
-.. include:: ../includes/wasm-notavail.rst
-
-.. class:: async_chat()
-
- This class is an abstract subclass of :class:`asyncore.dispatcher`. To make
- practical use of the code you must subclass :class:`async_chat`, providing
- meaningful :meth:`collect_incoming_data` and :meth:`found_terminator`
- methods.
- The :class:`asyncore.dispatcher` methods can be used, although not all make
- sense in a message/response context.
-
- Like :class:`asyncore.dispatcher`, :class:`async_chat` defines a set of
- events that are generated by an analysis of socket conditions after a
- :c:func:`select` call. Once the polling loop has been started the
- :class:`async_chat` object's methods are called by the event-processing
- framework with no action on the part of the programmer.
-
- Two class attributes can be modified, to improve performance, or possibly
- even to conserve memory.
-
-
- .. data:: ac_in_buffer_size
-
- The asynchronous input buffer size (default ``4096``).
-
-
- .. data:: ac_out_buffer_size
-
- The asynchronous output buffer size (default ``4096``).
-
- Unlike :class:`asyncore.dispatcher`, :class:`async_chat` allows you to
- define a :abbr:`FIFO (first-in, first-out)` queue of *producers*. A producer need
- have only one method, :meth:`more`, which should return data to be
- transmitted on the channel.
- The producer indicates exhaustion (*i.e.* that it contains no more data) by
- having its :meth:`more` method return the empty bytes object. At this point
- the :class:`async_chat` object removes the producer from the queue and starts
- using the next producer, if any. When the producer queue is empty the
- :meth:`handle_write` method does nothing. You use the channel object's
- :meth:`set_terminator` method to describe how to recognize the end of, or
- an important breakpoint in, an incoming transmission from the remote
- endpoint.
-
- To build a functioning :class:`async_chat` subclass your input methods
- :meth:`collect_incoming_data` and :meth:`found_terminator` must handle the
- data that the channel receives asynchronously. The methods are described
- below.
-
-
-.. method:: async_chat.close_when_done()
-
- Pushes a ``None`` on to the producer queue. When this producer is popped off
- the queue it causes the channel to be closed.
-
-
-.. method:: async_chat.collect_incoming_data(data)
-
- Called with *data* holding an arbitrary amount of received data. The
- default method, which must be overridden, raises a
- :exc:`NotImplementedError` exception.
-
-
-.. method:: async_chat.discard_buffers()
-
- In emergencies this method will discard any data held in the input and/or
- output buffers and the producer queue.
-
-
-.. method:: async_chat.found_terminator()
-
- Called when the incoming data stream matches the termination condition set
- by :meth:`set_terminator`. The default method, which must be overridden,
- raises a :exc:`NotImplementedError` exception. The buffered input data
- should be available via an instance attribute.
-
-
-.. method:: async_chat.get_terminator()
-
- Returns the current terminator for the channel.
-
-
-.. method:: async_chat.push(data)
-
- Pushes data on to the channel's queue to ensure its transmission.
- This is all you need to do to have the channel write the data out to the
- network, although it is possible to use your own producers in more complex
- schemes to implement encryption and chunking, for example.
-
-
-.. method:: async_chat.push_with_producer(producer)
-
- Takes a producer object and adds it to the producer queue associated with
- the channel. When all currently pushed producers have been exhausted the
- channel will consume this producer's data by calling its :meth:`more`
- method and send the data to the remote endpoint.
-
-
-.. method:: async_chat.set_terminator(term)
-
- Sets the terminating condition to be recognized on the channel. ``term``
- may be any of three types of value, corresponding to three different ways
- to handle incoming protocol data.
-
- +-----------+---------------------------------------------+
- | term | Description |
- +===========+=============================================+
- | *string* | Will call :meth:`found_terminator` when the |
- | | string is found in the input stream |
- +-----------+---------------------------------------------+
- | *integer* | Will call :meth:`found_terminator` when the |
- | | indicated number of characters have been |
- | | received |
- +-----------+---------------------------------------------+
- | ``None`` | The channel continues to collect data |
- | | forever |
- +-----------+---------------------------------------------+
-
- Note that any data following the terminator will be available for reading
- by the channel after :meth:`found_terminator` is called.
-
-
-.. _asynchat-example:
-
-asynchat Example
-----------------
-
-The following partial example shows how HTTP requests can be read with
-:class:`async_chat`. A web server might create an
-:class:`http_request_handler` object for each incoming client connection.
-Notice that initially the channel terminator is set to match the blank line at
-the end of the HTTP headers, and a flag indicates that the headers are being
-read.
-
-Once the headers have been read, if the request is of type POST (indicating
-that further data are present in the input stream) then the
-``Content-Length:`` header is used to set a numeric terminator to read the
-right amount of data from the channel.
-
-The :meth:`handle_request` method is called once all relevant input has been
-marshalled, after setting the channel terminator to ``None`` to ensure that
-any extraneous data sent by the web client are ignored. ::
-
-
- import asynchat
-
- class http_request_handler(asynchat.async_chat):
-
- def __init__(self, sock, addr, sessions, log):
- asynchat.async_chat.__init__(self, sock=sock)
- self.addr = addr
- self.sessions = sessions
- self.ibuffer = []
- self.obuffer = b""
- self.set_terminator(b"\r\n\r\n")
- self.reading_headers = True
- self.handling = False
- self.cgi_data = None
- self.log = log
-
- def collect_incoming_data(self, data):
- """Buffer the data"""
- self.ibuffer.append(data)
-
- def found_terminator(self):
- if self.reading_headers:
- self.reading_headers = False
- self.parse_headers(b"".join(self.ibuffer))
- self.ibuffer = []
- if self.op.upper() == b"POST":
- clen = self.headers.getheader("content-length")
- self.set_terminator(int(clen))
- else:
- self.handling = True
- self.set_terminator(None)
- self.handle_request()
- elif not self.handling:
- self.set_terminator(None) # browsers sometimes over-send
- self.cgi_data = parse(self.headers, b"".join(self.ibuffer))
- self.handling = True
- self.ibuffer = []
- self.handle_request()
diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
index 921a394a59fec7c..a9c3a0183bb72d2 100644
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -34,7 +34,7 @@ There are several ways to enable asyncio debug mode:
In addition to enabling the debug mode, consider also:
* setting the log level of the :ref:`asyncio logger ` to
- :py:data:`logging.DEBUG`, for example the following snippet of code
+ :py:const:`logging.DEBUG`, for example the following snippet of code
can be run at startup of the application::
logging.basicConfig(level=logging.DEBUG)
@@ -99,7 +99,7 @@ To schedule a coroutine object from a different OS thread, the
# Wait for the result:
result = future.result()
-To handle signals and to execute subprocesses, the event loop must be
+To handle signals the event loop must be
run in the main thread.
The :meth:`loop.run_in_executor` method can be used with a
@@ -142,7 +142,7 @@ Logging
asyncio uses the :mod:`logging` module and all logging is performed
via the ``"asyncio"`` logger.
-The default log level is :py:data:`logging.INFO`, which can be easily
+The default log level is :py:const:`logging.INFO`, which can be easily
adjusted::
logging.getLogger("asyncio").setLevel(logging.WARNING)
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index d0a1ed2b99e55d1..ea1d146f06cf2bd 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -33,7 +33,8 @@ an event loop:
Return the running event loop in the current OS thread.
- If there is no running event loop a :exc:`RuntimeError` is raised.
+ Raise a :exc:`RuntimeError` if there is no running event loop.
+
This function can only be called from a coroutine or a callback.
.. versionadded:: 3.7
@@ -42,27 +43,29 @@ an event loop:
Get the current event loop.
- If there is no current event loop set in the current OS thread,
- the OS thread is main, and :func:`set_event_loop` has not yet
- been called, asyncio will create a new event loop and set it as the
- current one.
+ When called from a coroutine or a callback (e.g. scheduled with
+ call_soon or similar API), this function will always return the
+ running event loop.
+
+ If there is no running event loop set, the function will return
+ the result of the ``get_event_loop_policy().get_event_loop()`` call.
Because this function has rather complex behavior (especially
when custom event loop policies are in use), using the
:func:`get_running_loop` function is preferred to :func:`get_event_loop`
in coroutines and callbacks.
- Consider also using the :func:`asyncio.run` function instead of using
- lower level functions to manually create and close an event loop.
+ As noted above, consider using the higher-level :func:`asyncio.run` function,
+ instead of using these lower level functions to manually create and close an
+ event loop.
- .. deprecated:: 3.10
- Deprecation warning is emitted if there is no running event loop.
- In future Python releases, this function will be an alias of
- :func:`get_running_loop`.
+ .. deprecated:: 3.12
+ Deprecation warning is emitted if there is no current event loop.
+ In some future Python release this will become an error.
.. function:: set_event_loop(loop)
- Set *loop* as a current event loop for the current OS thread.
+ Set *loop* as the current event loop for the current OS thread.
.. function:: new_event_loop()
@@ -183,19 +186,24 @@ Running and stopping the loop
.. coroutinemethod:: loop.shutdown_default_executor(timeout=None)
Schedule the closure of the default executor and wait for it to join all of
- the threads in the :class:`ThreadPoolExecutor`. After calling this method, a
- :exc:`RuntimeError` will be raised if :meth:`loop.run_in_executor` is called
- while using the default executor.
+ the threads in the :class:`~concurrent.futures.ThreadPoolExecutor`.
+ Once this method has been called,
+ using the default executor with :meth:`loop.run_in_executor`
+ will raise a :exc:`RuntimeError`.
- The *timeout* parameter specifies the amount of time the executor will
- be given to finish joining. The default value is ``None``, which means the
- executor will be given an unlimited amount of time.
+ The *timeout* parameter specifies the amount of time
+ (in :class:`float` seconds) the executor will be given to finish joining.
+ With the default, ``None``,
+ the executor is allowed an unlimited amount of time.
- If the timeout duration is reached, a warning is emitted and executor is
- terminated without waiting for its threads to finish joining.
+ If the *timeout* is reached, a :exc:`RuntimeWarning` is emitted
+ and the default executor is terminated
+ without waiting for its threads to finish joining.
- Note that there is no need to call this function when
- :func:`asyncio.run` is used.
+ .. note::
+
+ Do not call this method when using :func:`asyncio.run`,
+ as the latter handles default executor shutdown automatically.
.. versionadded:: 3.9
@@ -210,22 +218,23 @@ Scheduling callbacks
Schedule the *callback* :term:`callback` to be called with
*args* arguments at the next iteration of the event loop.
+ Return an instance of :class:`asyncio.Handle`,
+ which can be used later to cancel the callback.
+
Callbacks are called in the order in which they are registered.
Each callback will be called exactly once.
- An optional keyword-only *context* argument allows specifying a
+ The optional keyword-only *context* argument specifies a
custom :class:`contextvars.Context` for the *callback* to run in.
- The current context is used when no *context* is provided.
-
- An instance of :class:`asyncio.Handle` is returned, which can be
- used later to cancel the callback.
+ Callbacks use the current context when no *context* is provided.
- This method is not thread-safe.
+ Unlike :meth:`call_soon_threadsafe`, this method is not thread-safe.
.. method:: loop.call_soon_threadsafe(callback, *args, context=None)
- A thread-safe variant of :meth:`call_soon`. Must be used to
- schedule callbacks *from another thread*.
+ A thread-safe variant of :meth:`call_soon`. When scheduling callbacks from
+ another thread, this function *must* be used, since :meth:`call_soon` is not
+ thread-safe.
Raises :exc:`RuntimeError` if called on a loop that's been closed.
This can happen on a secondary thread when the main application is
@@ -234,9 +243,9 @@ Scheduling callbacks
See the :ref:`concurrency and multithreading `
section of the documentation.
-.. versionchanged:: 3.7
- The *context* keyword-only parameter was added. See :pep:`567`
- for more details.
+ .. versionchanged:: 3.7
+ The *context* keyword-only parameter was added. See :pep:`567`
+ for more details.
.. _asyncio-pass-keywords:
@@ -394,11 +403,11 @@ Opening network connections
Open a streaming transport connection to a given
address specified by *host* and *port*.
- The socket family can be either :py:data:`~socket.AF_INET` or
- :py:data:`~socket.AF_INET6` depending on *host* (or the *family*
+ The socket family can be either :py:const:`~socket.AF_INET` or
+ :py:const:`~socket.AF_INET6` depending on *host* (or the *family*
argument, if provided).
- The socket type will be :py:data:`~socket.SOCK_STREAM`.
+ The socket type will be :py:const:`~socket.SOCK_STREAM`.
*protocol_factory* must be a callable returning an
:ref:`asyncio protocol ` implementation.
@@ -500,7 +509,7 @@ Opening network connections
.. versionchanged:: 3.6
- The socket option :py:data:`~socket.TCP_NODELAY` is set by default
+ The socket option :ref:`socket.TCP_NODELAY ` is set by default
for all TCP connections.
.. versionchanged:: 3.7
@@ -515,12 +524,12 @@ Opening network connections
When a server's IPv4 path and protocol are working, but the server's
IPv6 path and protocol are not working, a dual-stack client
application experiences significant connection delay compared to an
- IPv4-only client. This is undesirable because it causes the dual-
- stack client to have a worse user experience. This document
+ IPv4-only client. This is undesirable because it causes the
+ dual-stack client to have a worse user experience. This document
specifies requirements for algorithms that reduce this user-visible
delay and provides an algorithm.
- For more information: https://tools.ietf.org/html/rfc6555
+ For more information: https://datatracker.ietf.org/doc/html/rfc6555
.. versionchanged:: 3.11
@@ -543,11 +552,11 @@ Opening network connections
Create a datagram connection.
- The socket family can be either :py:data:`~socket.AF_INET`,
- :py:data:`~socket.AF_INET6`, or :py:data:`~socket.AF_UNIX`,
+ The socket family can be either :py:const:`~socket.AF_INET`,
+ :py:const:`~socket.AF_INET6`, or :py:const:`~socket.AF_UNIX`,
depending on *host* (or the *family* argument, if provided).
- The socket type will be :py:data:`~socket.SOCK_DGRAM`.
+ The socket type will be :py:const:`~socket.SOCK_DGRAM`.
*protocol_factory* must be a callable returning a
:ref:`protocol ` implementation.
@@ -572,7 +581,7 @@ Opening network connections
* *reuse_port* tells the kernel to allow this endpoint to be bound to the
same port as other existing endpoints are bound to, so long as they all
set this flag when being created. This option is not supported on Windows
- and some Unixes. If the :py:data:`~socket.SO_REUSEPORT` constant is not
+ and some Unixes. If the :ref:`socket.SO_REUSEPORT ` constant is not
defined then this capability is unsupported.
* *allow_broadcast* tells the kernel to allow this endpoint to send
@@ -598,7 +607,8 @@ Opening network connections
.. versionchanged:: 3.8.1
The *reuse_address* parameter is no longer supported, as using
- :py:data:`~sockets.SO_REUSEADDR` poses a significant security concern for
+ :ref:`socket.SO_REUSEADDR `
+ poses a significant security concern for
UDP. Explicitly passing ``reuse_address=True`` will raise an exception.
When multiple processes with differing UIDs assign sockets to an
@@ -607,7 +617,8 @@ Opening network connections
For supported platforms, *reuse_port* can be used as a replacement for
similar functionality. With *reuse_port*,
- :py:data:`~sockets.SO_REUSEPORT` is used instead, which specifically
+ :ref:`socket.SO_REUSEPORT `
+ is used instead, which specifically
prevents processes with differing UIDs from assigning sockets to the same
socket address.
@@ -625,8 +636,8 @@ Opening network connections
Create a Unix connection.
- The socket family will be :py:data:`~socket.AF_UNIX`; socket
- type will be :py:data:`~socket.SOCK_STREAM`.
+ The socket family will be :py:const:`~socket.AF_UNIX`; socket
+ type will be :py:const:`~socket.SOCK_STREAM`.
A tuple of ``(transport, protocol)`` is returned on success.
@@ -652,6 +663,8 @@ Opening network connections
Creating network servers
^^^^^^^^^^^^^^^^^^^^^^^^
+.. _loop_create_server:
+
.. coroutinemethod:: loop.create_server(protocol_factory, \
host=None, port=None, *, \
family=socket.AF_UNSPEC, \
@@ -662,7 +675,7 @@ Creating network servers
ssl_shutdown_timeout=None, \
start_serving=True)
- Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) listening
+ Create a TCP server (socket type :const:`~socket.SOCK_STREAM`) listening
on *port* of the *host* address.
Returns a :class:`Server` object.
@@ -690,10 +703,10 @@ Creating network servers
be selected (note that if *host* resolves to multiple network interfaces,
a different random port will be selected for each interface).
- * *family* can be set to either :data:`socket.AF_INET` or
- :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6.
+ * *family* can be set to either :const:`socket.AF_INET` or
+ :const:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6.
If not set, the *family* will be determined from host name
- (defaults to :data:`~socket.AF_UNSPEC`).
+ (defaults to :const:`~socket.AF_UNSPEC`).
* *flags* is a bitmask for :meth:`getaddrinfo`.
@@ -747,7 +760,7 @@ Creating network servers
.. versionchanged:: 3.6
Added *ssl_handshake_timeout* and *start_serving* parameters.
- The socket option :py:data:`~socket.TCP_NODELAY` is set by default
+ The socket option :ref:`socket.TCP_NODELAY ` is set by default
for all TCP connections.
.. versionchanged:: 3.11
@@ -765,16 +778,20 @@ Creating network servers
*, sock=None, backlog=100, ssl=None, \
ssl_handshake_timeout=None, \
ssl_shutdown_timeout=None, \
- start_serving=True)
+ start_serving=True, cleanup_socket=True)
Similar to :meth:`loop.create_server` but works with the
- :py:data:`~socket.AF_UNIX` socket family.
+ :py:const:`~socket.AF_UNIX` socket family.
*path* is the name of a Unix domain socket, and is required,
unless a *sock* argument is provided. Abstract Unix sockets,
:class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths
are supported.
+ If *cleanup_socket* is True then the Unix socket will automatically
+ be removed from the filesystem when the server is closed, unless the
+ socket has been replaced after the server has been created.
+
See the documentation of the :meth:`loop.create_server` method
for information about arguments to this method.
@@ -789,6 +806,10 @@ Creating network servers
Added the *ssl_shutdown_timeout* parameter.
+ .. versionchanged:: 3.13
+
+ Added the *cleanup_socket* parameter.
+
.. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \
sock, *, ssl=None, ssl_handshake_timeout=None, \
@@ -886,6 +907,9 @@ TLS Upgrade
object only because the coder caches *protocol*-side data and sporadically
exchanges extra TLS session packets with *transport*.
+ In some situations (e.g. when the passed transport is already closing) this
+ may return ``None``.
+
Parameters:
* *transport* and *protocol* instances that methods like
@@ -927,7 +951,8 @@ Watching file descriptors
.. method:: loop.remove_reader(fd)
- Stop monitoring the *fd* file descriptor for read availability.
+ Stop monitoring the *fd* file descriptor for read availability. Returns
+ ``True`` if *fd* was previously being monitored for reads.
.. method:: loop.add_writer(fd, callback, *args)
@@ -940,7 +965,8 @@ Watching file descriptors
.. method:: loop.remove_writer(fd)
- Stop monitoring the *fd* file descriptor for write availability.
+ Stop monitoring the *fd* file descriptor for write availability. Returns
+ ``True`` if *fd* was previously being monitored for writes.
See also :ref:`Platform Support ` section
for some limitations of these methods.
@@ -1177,6 +1203,8 @@ Working with pipes
Unix signals
^^^^^^^^^^^^
+.. _loop_add_signal_handler:
+
.. method:: loop.add_signal_handler(signum, callback, *args)
Set *callback* as the handler for the *signum* signal.
@@ -1377,6 +1405,14 @@ Enabling debug mode
The new :ref:`Python Development Mode ` can now also be used
to enable the debug mode.
+.. attribute:: loop.slow_callback_duration
+
+ This attribute can be used to set the
+ minimum execution duration in seconds that is considered "slow".
+ When debug mode is enabled, "slow" callbacks are logged.
+
+ Default value is 100 milliseconds.
+
.. seealso::
The :ref:`debug mode of asyncio `.
@@ -1397,6 +1433,8 @@ async/await code consider using the high-level
:ref:`Subprocess Support on Windows ` for
details.
+.. _loop_subprocess_exec:
+
.. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \
stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
stderr=subprocess.PIPE, **kwargs)
@@ -1427,9 +1465,8 @@ async/await code consider using the high-level
* *stdin* can be any of these:
- * a file-like object representing a pipe to be connected to the
- subprocess's standard input stream using
- :meth:`~loop.connect_write_pipe`
+ * a file-like object
+ * an existing file descriptor (a positive integer), for example those created with :meth:`os.pipe()`
* the :const:`subprocess.PIPE` constant (default) which will create a new
pipe and connect it,
* the value ``None`` which will make the subprocess inherit the file
@@ -1439,9 +1476,7 @@ async/await code consider using the high-level
* *stdout* can be any of these:
- * a file-like object representing a pipe to be connected to the
- subprocess's standard output stream using
- :meth:`~loop.connect_write_pipe`
+ * a file-like object
* the :const:`subprocess.PIPE` constant (default) which will create a new
pipe and connect it,
* the value ``None`` which will make the subprocess inherit the file
@@ -1451,9 +1486,7 @@ async/await code consider using the high-level
* *stderr* can be any of these:
- * a file-like object representing a pipe to be connected to the
- subprocess's standard error stream using
- :meth:`~loop.connect_write_pipe`
+ * a file-like object
* the :const:`subprocess.PIPE` constant (default) which will create a new
pipe and connect it,
* the value ``None`` which will make the subprocess inherit the file
@@ -1472,6 +1505,11 @@ async/await code consider using the high-level
as text. :func:`bytes.decode` can be used to convert the bytes returned
from the stream to text.
+ If a file-like object passed as *stdin*, *stdout* or *stderr* represents a
+ pipe, then the other side of this pipe should be registered with
+ :meth:`~loop.connect_write_pipe` or :meth:`~loop.connect_read_pipe` for use
+ with the event loop.
+
See the constructor of the :class:`subprocess.Popen` class
for documentation on other arguments.
@@ -1560,7 +1598,7 @@ Server objects are created by :meth:`loop.create_server`,
:meth:`loop.create_unix_server`, :func:`start_server`,
and :func:`start_unix_server` functions.
-Do not instantiate the class directly.
+Do not instantiate the :class:`Server` class directly.
.. class:: Server
@@ -1580,6 +1618,9 @@ Do not instantiate the class directly.
.. versionchanged:: 3.7
Server object is an asynchronous context manager since Python 3.7.
+ .. versionchanged:: 3.11
+ This class was exposed publicly as ``asyncio.Server`` in Python 3.9.11, 3.10.3 and 3.11.
+
.. method:: close()
Stop serving: close listening sockets and set the :attr:`sockets`
@@ -1588,8 +1629,9 @@ Do not instantiate the class directly.
The sockets that represent existing incoming client connections
are left open.
- The server is closed asynchronously, use the :meth:`wait_closed`
- coroutine to wait until the server is closed.
+ The server is closed asynchronously; use the :meth:`wait_closed`
+ coroutine to wait until the server is closed (and no more
+ connections are active).
.. method:: get_loop()
@@ -1647,11 +1689,13 @@ Do not instantiate the class directly.
.. coroutinemethod:: wait_closed()
- Wait until the :meth:`close` method completes.
+ Wait until the :meth:`close` method completes and all active
+ connections have finished.
.. attribute:: sockets
- List of :class:`socket.socket` objects the server is listening on.
+ List of socket-like objects, ``asyncio.trsock.TransportSocket``, which
+ the server is listening on.
.. versionchanged:: 3.7
Prior to Python 3.7 ``Server.sockets`` used to return an
@@ -1668,13 +1712,13 @@ Event Loop Implementations
asyncio ships with two different event loop implementations:
:class:`SelectorEventLoop` and :class:`ProactorEventLoop`.
-By default asyncio is configured to use :class:`SelectorEventLoop`
-on Unix and :class:`ProactorEventLoop` on Windows.
+By default asyncio is configured to use :class:`EventLoop`.
.. class:: SelectorEventLoop
- An event loop based on the :mod:`selectors` module.
+ A subclass of :class:`AbstractEventLoop` based on the
+ :mod:`selectors` module.
Uses the most efficient *selector* available for the given
platform. It is also possible to manually configure the
@@ -1696,7 +1740,7 @@ on Unix and :class:`ProactorEventLoop` on Windows.
.. class:: ProactorEventLoop
- An event loop for Windows that uses "I/O Completion Ports" (IOCP).
+ A subclass of :class:`AbstractEventLoop` for Windows that uses "I/O Completion Ports" (IOCP).
.. availability:: Windows.
@@ -1705,6 +1749,14 @@ on Unix and :class:`ProactorEventLoop` on Windows.
`MSDN documentation on I/O Completion Ports
`_.
+.. class:: EventLoop
+
+ An alias to the most efficient available subclass of :class:`AbstractEventLoop` for the given
+ platform.
+
+ It is an alias to :class:`SelectorEventLoop` on Unix and :class:`ProactorEventLoop` on Windows.
+
+ .. versionadded:: 3.13
.. class:: AbstractEventLoop
@@ -1854,7 +1906,7 @@ Set signal handlers for SIGINT and SIGTERM
(This ``signals`` example only works on Unix.)
-Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM`
+Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM`
using the :meth:`loop.add_signal_handler` method::
import asyncio
diff --git a/Doc/library/asyncio-exceptions.rst b/Doc/library/asyncio-exceptions.rst
index 9250f01b8a08953..7ad9103ca3fdfc8 100644
--- a/Doc/library/asyncio-exceptions.rst
+++ b/Doc/library/asyncio-exceptions.rst
@@ -31,7 +31,7 @@ Exceptions
.. versionchanged:: 3.8
- :exc:`CancelledError` is now a subclass of :class:`BaseException`.
+ :exc:`CancelledError` is now a subclass of :class:`BaseException` rather than :class:`Exception`.
.. exception:: InvalidStateError
diff --git a/Doc/library/asyncio-extending.rst b/Doc/library/asyncio-extending.rst
index 8ffd356f2d1cc3b..e7b293f484f8de6 100644
--- a/Doc/library/asyncio-extending.rst
+++ b/Doc/library/asyncio-extending.rst
@@ -69,7 +69,7 @@ Task lifetime support
=====================
A third party task implementation should call the following functions to keep a task
-visible by :func:`asyncio.get_tasks` and :func:`asyncio.current_task`:
+visible by :func:`asyncio.all_tasks` and :func:`asyncio.current_task`:
.. function:: _register_task(task)
diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst
index 70cec9b2f90248d..893ae5518f757d8 100644
--- a/Doc/library/asyncio-future.rst
+++ b/Doc/library/asyncio-future.rst
@@ -276,4 +276,4 @@ the Future has a result::
:func:`concurrent.futures.as_completed` functions.
- :meth:`asyncio.Future.cancel` accepts an optional ``msg`` argument,
- but :func:`concurrent.futures.cancel` does not.
+ but :meth:`concurrent.futures.Future.cancel` does not.
diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst
index b7ad888a7b67ab5..67136ba69ec875b 100644
--- a/Doc/library/asyncio-llapi-index.rst
+++ b/Doc/library/asyncio-llapi-index.rst
@@ -19,7 +19,7 @@ Obtaining the Event Loop
- The **preferred** function to get the running event loop.
* - :func:`asyncio.get_event_loop`
- - Get an event loop instance (current or via the policy).
+ - Get an event loop instance (running or current via the current policy).
* - :func:`asyncio.set_event_loop`
- Set the event loop as current via the current policy.
@@ -484,19 +484,19 @@ Protocol classes can implement the following **callback methods**:
:widths: 50 50
:class: full-width-table
- * - ``callback`` :meth:`pipe_data_received()
- `
+ * - ``callback`` :meth:`~SubprocessProtocol.pipe_data_received`
- Called when the child process writes data into its
*stdout* or *stderr* pipe.
- * - ``callback`` :meth:`pipe_connection_lost()
- `
+ * - ``callback`` :meth:`~SubprocessProtocol.pipe_connection_lost`
- Called when one of the pipes communicating with
the child process is closed.
* - ``callback`` :meth:`process_exited()
`
- - Called when the child process has exited.
+ - Called when the child process has exited. It can be called before
+ :meth:`~SubprocessProtocol.pipe_data_received` and
+ :meth:`~SubprocessProtocol.pipe_connection_lost` methods.
Event Loop Policies
diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst
index 50ad8a2ab70324a..19ec726c1be0602 100644
--- a/Doc/library/asyncio-platforms.rst
+++ b/Doc/library/asyncio-platforms.rst
@@ -37,7 +37,7 @@ All event loops on Windows do not support the following methods:
* :meth:`loop.create_unix_connection` and
:meth:`loop.create_unix_server` are not supported.
- The :data:`socket.AF_UNIX` socket family is specific to Unix.
+ The :const:`socket.AF_UNIX` socket family is specific to Unix.
* :meth:`loop.add_signal_handler` and
:meth:`loop.remove_signal_handler` are not supported.
diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst
index 052378ef32743b5..0d7821e608ec983 100644
--- a/Doc/library/asyncio-policy.rst
+++ b/Doc/library/asyncio-policy.rst
@@ -116,6 +116,12 @@ asyncio ships with the following built-in policies:
On Windows, :class:`ProactorEventLoop` is now used by default.
+ .. deprecated:: 3.12
+ The :meth:`get_event_loop` method of the default asyncio policy now emits
+ a :exc:`DeprecationWarning` if there is no current event loop set and it
+ decides to create one.
+ In some future Python release this will become an error.
+
.. class:: WindowsSelectorEventLoopPolicy
@@ -222,6 +228,9 @@ implementation used by the asyncio event loop:
This method has to be called to ensure that underlying
resources are cleaned-up.
+ .. deprecated:: 3.12
+
+
.. class:: ThreadedChildWatcher
This implementation starts a new waiting thread for every subprocess spawn.
diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index 7bc906eaafc1f26..3f734f544afe216 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -708,6 +708,9 @@ factories passed to the :meth:`loop.subprocess_exec` and
Called when the child process has exited.
+ It can be called before :meth:`~SubprocessProtocol.pipe_data_received` and
+ :meth:`~SubprocessProtocol.pipe_connection_lost` methods.
+
Examples
========
@@ -746,7 +749,7 @@ received data, and close the connection::
loop = asyncio.get_running_loop()
server = await loop.create_server(
- lambda: EchoServerProtocol(),
+ EchoServerProtocol,
'127.0.0.1', 8888)
async with server:
@@ -850,7 +853,7 @@ method, sends back received data::
# One protocol instance will be created to serve all
# client requests.
transport, protocol = await loop.create_datagram_endpoint(
- lambda: EchoServerProtocol(),
+ EchoServerProtocol,
local_addr=('127.0.0.1', 9999))
try:
@@ -1003,12 +1006,26 @@ The subprocess is created by the :meth:`loop.subprocess_exec` method::
def __init__(self, exit_future):
self.exit_future = exit_future
self.output = bytearray()
+ self.pipe_closed = False
+ self.exited = False
+
+ def pipe_connection_lost(self, fd, exc):
+ self.pipe_closed = True
+ self.check_for_exit()
def pipe_data_received(self, fd, data):
self.output.extend(data)
def process_exited(self):
- self.exit_future.set_result(True)
+ self.exited = True
+ # process_exited() method can be called before
+ # pipe_connection_lost() method: wait until both methods are
+ # called.
+ self.check_for_exit()
+
+ def check_for_exit(self):
+ if self.pipe_closed and self.exited:
+ self.exit_future.set_result(True)
async def get_date():
# Get a reference to the event loop as we plan to use
diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst
index c43d664eba717f2..ec170dfde9e9aa2 100644
--- a/Doc/library/asyncio-runner.rst
+++ b/Doc/library/asyncio-runner.rst
@@ -22,7 +22,7 @@ to simplify async code usage for common wide-spread scenarios.
Running an asyncio Program
==========================
-.. function:: run(coro, *, debug=None)
+.. function:: run(coro, *, debug=None, loop_factory=None)
Execute the :term:`coroutine` *coro* and return the result.
@@ -37,9 +37,13 @@ Running an asyncio Program
debug mode explicitly. ``None`` is used to respect the global
:ref:`asyncio-debug-mode` settings.
- This function always creates a new event loop and closes it at
- the end. It should be used as a main entry point for asyncio
- programs, and should ideally only be called once.
+ If *loop_factory* is not ``None``, it is used to create a new event loop;
+ otherwise :func:`asyncio.new_event_loop` is used. The loop is closed at the end.
+ This function should be used as a main entry point for asyncio programs,
+ and should ideally only be called once. It is recommended to use
+ *loop_factory* to configure the event loop instead of policies.
+ Passing :class:`asyncio.EventLoop` allows running asyncio without the
+ policy system.
The executor is given a timeout duration of 5 minutes to shutdown.
If the executor hasn't finished within that duration, a warning is
@@ -62,6 +66,10 @@ Running an asyncio Program
*debug* is ``None`` by default to respect the global debug mode settings.
+ .. versionchanged:: 3.12
+
+ Added *loop_factory* parameter.
+
Runner context manager
======================
diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst
index d87e3c042c99770..0736e783bbc8c8e 100644
--- a/Doc/library/asyncio-stream.rst
+++ b/Doc/library/asyncio-stream.rst
@@ -52,6 +52,7 @@ and work with streams:
limit=None, ssl=None, family=0, proto=0, \
flags=0, sock=None, local_addr=None, \
server_hostname=None, ssl_handshake_timeout=None, \
+ ssl_shutdown_timeout=None, \
happy_eyeballs_delay=None, interleave=None)
Establish a network connection and return a pair of
@@ -82,6 +83,9 @@ and work with streams:
.. versionchanged:: 3.10
Removed the *loop* parameter.
+ .. versionchanged:: 3.11
+ Added the *ssl_shutdown_timeout* parameter.
+
.. coroutinefunction:: start_server(client_connected_cb, host=None, \
port=None, *, limit=None, \
@@ -89,7 +93,7 @@ and work with streams:
flags=socket.AI_PASSIVE, sock=None, \
backlog=100, ssl=None, reuse_address=None, \
reuse_port=None, ssl_handshake_timeout=None, \
- start_serving=True)
+ ssl_shutdown_timeout=None, start_serving=True)
Start a socket server.
@@ -121,12 +125,15 @@ and work with streams:
.. versionchanged:: 3.10
Removed the *loop* parameter.
+ .. versionchanged:: 3.11
+ Added the *ssl_shutdown_timeout* parameter.
+
.. rubric:: Unix Sockets
.. coroutinefunction:: open_unix_connection(path=None, *, limit=None, \
ssl=None, sock=None, server_hostname=None, \
- ssl_handshake_timeout=None)
+ ssl_handshake_timeout=None, ssl_shutdown_timeout=None)
Establish a Unix socket connection and return a pair of
``(reader, writer)``.
@@ -150,10 +157,14 @@ and work with streams:
.. versionchanged:: 3.10
Removed the *loop* parameter.
+ .. versionchanged:: 3.11
+ Added the *ssl_shutdown_timeout* parameter.
+
.. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \
*, limit=None, sock=None, backlog=100, ssl=None, \
- ssl_handshake_timeout=None, start_serving=True)
+ ssl_handshake_timeout=None, \
+ ssl_shutdown_timeout=None, start_serving=True)
Start a Unix socket server.
@@ -176,6 +187,9 @@ and work with streams:
.. versionchanged:: 3.10
Removed the *loop* parameter.
+ .. versionchanged:: 3.11
+ Added the *ssl_shutdown_timeout* parameter.
+
StreamReader
============
@@ -190,14 +204,26 @@ StreamReader
directly; use :func:`open_connection` and :func:`start_server`
instead.
+ .. method:: feed_eof()
+
+ Acknowledge the EOF.
+
.. coroutinemethod:: read(n=-1)
- Read up to *n* bytes. If *n* is not provided, or set to ``-1``,
- read until EOF and return all read bytes.
+ Read up to *n* bytes from the stream.
+ If *n* is not provided or set to ``-1``,
+ read until EOF, then return all read :class:`bytes`.
If EOF was received and the internal buffer is empty,
return an empty ``bytes`` object.
+ If *n* is ``0``, return an empty ``bytes`` object immediately.
+
+ If *n* is positive, return at most *n* available ``bytes``
+ as soon as at least 1 byte is available in the internal buffer.
+ If EOF is received before any byte is read, return an empty
+ ``bytes`` object.
+
.. coroutinemethod:: readline()
Read one line, where "line" is a sequence of bytes
@@ -281,7 +307,8 @@ StreamWriter
The method closes the stream and the underlying socket.
- The method should be used along with the ``wait_closed()`` method::
+ The method should be used, though not mandatory,
+ along with the ``wait_closed()`` method::
stream.close()
await stream.wait_closed()
@@ -321,7 +348,7 @@ StreamWriter
returns immediately.
.. coroutinemethod:: start_tls(sslcontext, \*, server_hostname=None, \
- ssl_handshake_timeout=None)
+ ssl_handshake_timeout=None, ssl_shutdown_timeout=None)
Upgrade an existing stream-based connection to TLS.
@@ -336,8 +363,16 @@ StreamWriter
handshake to complete before aborting the connection. ``60.0`` seconds
if ``None`` (default).
+ * *ssl_shutdown_timeout* is the time in seconds to wait for the SSL shutdown
+ to complete before aborting the connection. ``30.0`` seconds if ``None``
+ (default).
+
.. versionadded:: 3.11
+ .. versionchanged:: 3.12
+ Added the *ssl_shutdown_timeout* parameter.
+
+
.. method:: is_closing()
Return ``True`` if the stream is closed or in the process of
@@ -350,7 +385,8 @@ StreamWriter
Wait until the stream is closed.
Should be called after :meth:`close` to wait until the underlying
- connection is closed.
+ connection is closed, ensuring that all data has been flushed
+ before e.g. exiting the program.
.. versionadded:: 3.7
@@ -380,6 +416,7 @@ TCP echo client using the :func:`asyncio.open_connection` function::
print('Close the connection')
writer.close()
+ await writer.wait_closed()
asyncio.run(tcp_echo_client('Hello World!'))
@@ -412,6 +449,7 @@ TCP echo server using the :func:`asyncio.start_server` function::
print("Close the connection")
writer.close()
+ await writer.wait_closed()
async def main():
server = await asyncio.start_server(
@@ -468,6 +506,7 @@ Simple example querying HTTP headers of the URL passed on the command line::
# Ignore the body, close the socket
writer.close()
+ await writer.wait_closed()
url = sys.argv[1]
asyncio.run(print_http_headers(url))
@@ -513,6 +552,7 @@ Coroutine waiting until a socket receives data using the
# Got data, we are done: close the socket
print("Received:", data.decode())
writer.close()
+ await writer.wait_closed()
# Close the second socket
wsock.close()
diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst
index 28d0b21e8180b64..bf35b1cb798aee1 100644
--- a/Doc/library/asyncio-subprocess.rst
+++ b/Doc/library/asyncio-subprocess.rst
@@ -68,7 +68,7 @@ Creating Subprocesses
The *limit* argument sets the buffer limit for :class:`StreamReader`
wrappers for :attr:`Process.stdout` and :attr:`Process.stderr`
- (if :attr:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments).
+ (if :const:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments).
Return a :class:`~asyncio.subprocess.Process` instance.
@@ -86,7 +86,7 @@ Creating Subprocesses
The *limit* argument sets the buffer limit for :class:`StreamReader`
wrappers for :attr:`Process.stdout` and :attr:`Process.stderr`
- (if :attr:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments).
+ (if :const:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments).
Return a :class:`~asyncio.subprocess.Process` instance.
@@ -175,7 +175,7 @@ their completion.
* the :meth:`~asyncio.subprocess.Process.communicate` and
:meth:`~asyncio.subprocess.Process.wait` methods don't have a
- *timeout* parameter: use the :func:`wait_for` function;
+ *timeout* parameter: use the :func:`~asyncio.wait_for` function;
* the :meth:`Process.wait() ` method
is asynchronous, whereas :meth:`subprocess.Popen.wait` method
@@ -207,8 +207,9 @@ their completion.
Interact with process:
1. send data to *stdin* (if *input* is not ``None``);
- 2. read data from *stdout* and *stderr*, until EOF is reached;
- 3. wait for process to terminate.
+ 2. closes *stdin*;
+ 3. read data from *stdout* and *stderr*, until EOF is reached;
+ 4. wait for process to terminate.
The optional *input* argument is the data (:class:`bytes` object)
that will be sent to the child process.
@@ -229,6 +230,10 @@ their completion.
Note, that the data read is buffered in memory, so do not use
this method if the data size is large or unlimited.
+ .. versionchanged:: 3.12
+
+ *stdin* gets closed when `input=None` too.
+
.. method:: send_signal(signal)
Sends the signal *signal* to the child process.
@@ -244,7 +249,7 @@ their completion.
Stop the child process.
- On POSIX systems this method sends :py:data:`signal.SIGTERM` to the
+ On POSIX systems this method sends :py:const:`signal.SIGTERM` to the
child process.
On Windows the Win32 API function :c:func:`TerminateProcess` is
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index fb6d23fda03e41d..797065c8ccf8941 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -18,6 +18,10 @@ and Tasks.
Coroutines
==========
+**Source code:** :source:`Lib/asyncio/coroutines.py`
+
+----------------------------------------------------
+
:term:`Coroutines ` declared with the async/await syntax is the
preferred way of writing asyncio applications. For example, the following
snippet of code prints "hello", waits 1 second,
@@ -117,7 +121,7 @@ To actually run a coroutine, asyncio provides the following mechanisms:
print(f"started at {time.strftime('%X')}")
- # The wait is implicit when the context manager exits.
+ # The await is implicit when the context manager exits.
print(f"finished at {time.strftime('%X')}")
@@ -230,6 +234,10 @@ is :meth:`loop.run_in_executor`.
Creating Tasks
==============
+**Source code:** :source:`Lib/asyncio/tasks.py`
+
+-----------------------------------------------
+
.. function:: create_task(coro, *, name=None, context=None)
Wrap the *coro* :ref:`coroutine ` into a :class:`Task`
@@ -248,8 +256,9 @@ Creating Tasks
.. note::
- :meth:`asyncio.TaskGroup.create_task` is a newer alternative
- that allows for convenient waiting for a group of related tasks.
+ :meth:`asyncio.TaskGroup.create_task` is a new alternative
+ leveraging structural concurrency; it allows for waiting
+ for a group of related tasks with strong safety guarantees.
.. important::
@@ -292,13 +301,17 @@ in the task at the next opportunity.
It is recommended that coroutines use ``try/finally`` blocks to robustly
perform clean-up logic. In case :exc:`asyncio.CancelledError`
is explicitly caught, it should generally be propagated when
-clean-up is complete. Most code can safely ignore :exc:`asyncio.CancelledError`.
+clean-up is complete. :exc:`asyncio.CancelledError` directly subclasses
+:exc:`BaseException` so most code will not need to be aware of it.
The asyncio components that enable structured concurrency, like
:class:`asyncio.TaskGroup` and :func:`asyncio.timeout`,
are implemented using cancellation internally and might misbehave if
a coroutine swallows :exc:`asyncio.CancelledError`. Similarly, user code
-should not call :meth:`uncancel `.
+should not generally call :meth:`uncancel `.
+However, in cases when suppressing :exc:`asyncio.CancelledError` is
+truly desired, it is necessary to also call ``uncancel()`` to completely
+remove the cancellation state.
.. _taskgroups:
@@ -328,7 +341,7 @@ Example::
async with asyncio.TaskGroup() as tg:
task1 = tg.create_task(some_coro(...))
task2 = tg.create_task(another_coro(...))
- print("Both tasks have completed now.")
+ print(f"Both tasks have completed now: {task1.result()}, {task2.result()}")
The ``async with`` statement will wait for all tasks in the group to finish.
While waiting, new tasks may still be added to the group
@@ -413,6 +426,9 @@ Sleeping
.. versionchanged:: 3.10
Removed the *loop* parameter.
+ .. versionchanged:: 3.13
+ Raises :exc:`ValueError` if *delay* is :data:`~math.nan`.
+
Running Tasks Concurrently
==========================
@@ -447,8 +463,12 @@ Running Tasks Concurrently
Tasks/Futures to be cancelled.
.. note::
- A more modern way to create and run tasks concurrently and
- wait for their completion is :class:`asyncio.TaskGroup`.
+ A new alternative to create and run tasks concurrently and
+ wait for their completion is :class:`asyncio.TaskGroup`. *TaskGroup*
+ provides stronger safety guarantees than *gather* for scheduling a nesting of subtasks:
+ if a task (or a subtask, a task scheduled by a task)
+ raises an exception, *TaskGroup* will, while *gather* will not,
+ cancel the remaining scheduled tasks).
.. _asyncio_example_gather:
@@ -510,6 +530,51 @@ Running Tasks Concurrently
and there is no running event loop.
+.. _eager-task-factory:
+
+Eager Task Factory
+==================
+
+.. function:: eager_task_factory(loop, coro, *, name=None, context=None)
+
+ A task factory for eager task execution.
+
+ When using this factory (via :meth:`loop.set_task_factory(asyncio.eager_task_factory) `),
+ coroutines begin execution synchronously during :class:`Task` construction.
+ Tasks are only scheduled on the event loop if they block.
+ This can be a performance improvement as the overhead of loop scheduling
+ is avoided for coroutines that complete synchronously.
+
+ A common example where this is beneficial is coroutines which employ
+ caching or memoization to avoid actual I/O when possible.
+
+ .. note::
+
+ Immediate execution of the coroutine is a semantic change.
+ If the coroutine returns or raises, the task is never scheduled
+ to the event loop. If the coroutine execution blocks, the task is
+ scheduled to the event loop. This change may introduce behavior
+ changes to existing applications. For example,
+ the application's task execution order is likely to change.
+
+ .. versionadded:: 3.12
+
+.. function:: create_eager_task_factory(custom_task_constructor)
+
+ Create an eager task factory, similar to :func:`eager_task_factory`,
+ using the provided *custom_task_constructor* when creating a new task instead
+ of the default :class:`Task`.
+
+ *custom_task_constructor* must be a *callable* with the signature matching
+ the signature of :class:`Task.__init__ `.
+ The callable must return a :class:`asyncio.Task`-compatible object.
+
+ This function returns a *callable* intended to be used as a task factory of an
+ event loop via :meth:`loop.set_task_factory(factory) `).
+
+ .. versionadded:: 3.12
+
+
Shielding From Cancellation
===========================
@@ -566,9 +631,9 @@ Shielding From Cancellation
Timeouts
========
-.. coroutinefunction:: timeout(delay)
+.. function:: timeout(delay)
- An :ref:`asynchronous context manager `
+ Return an :ref:`asynchronous context manager `
that can be used to limit the amount of time spent waiting on
something.
@@ -589,16 +654,16 @@ Timeouts
If ``long_running_task`` takes more than 10 seconds to complete,
the context manager will cancel the current task and handle
the resulting :exc:`asyncio.CancelledError` internally, transforming it
- into an :exc:`asyncio.TimeoutError` which can be caught and handled.
+ into a :exc:`TimeoutError` which can be caught and handled.
.. note::
The :func:`asyncio.timeout` context manager is what transforms
- the :exc:`asyncio.CancelledError` into an :exc:`asyncio.TimeoutError`,
- which means the :exc:`asyncio.TimeoutError` can only be caught
+ the :exc:`asyncio.CancelledError` into a :exc:`TimeoutError`,
+ which means the :exc:`TimeoutError` can only be caught
*outside* of the context manager.
- Example of catching :exc:`asyncio.TimeoutError`::
+ Example of catching :exc:`TimeoutError`::
async def main():
try:
@@ -612,32 +677,26 @@ Timeouts
The context manager produced by :func:`asyncio.timeout` can be
rescheduled to a different deadline and inspected.
- .. class:: Timeout()
+ .. class:: Timeout(when)
An :ref:`asynchronous context manager `
- that limits time spent inside of it.
+ for cancelling overdue coroutines.
+
+ ``when`` should be an absolute time at which the context should time out,
+ as measured by the event loop's clock:
- .. versionadded:: 3.11
+ - If ``when`` is ``None``, the timeout will never trigger.
+ - If ``when < loop.time()``, the timeout will trigger on the next
+ iteration of the event loop.
.. method:: when() -> float | None
Return the current deadline, or ``None`` if the current
deadline is not set.
- The deadline is a float, consistent with the time returned by
- :meth:`loop.time`.
-
.. method:: reschedule(when: float | None)
- Change the time the timeout will trigger.
-
- If *when* is ``None``, any current deadline will be removed, and the
- context manager will wait indefinitely.
-
- If *when* is a float, it is set as the new deadline.
-
- if *when* is in the past, the timeout will trigger on the next
- iteration of the event loop.
+ Reschedule the timeout.
.. method:: expired() -> bool
@@ -658,14 +717,14 @@ Timeouts
except TimeoutError:
pass
- if cm.expired:
+ if cm.expired():
print("Looks like we haven't finished on time.")
Timeout context managers can be safely nested.
.. versionadded:: 3.11
-.. coroutinefunction:: timeout_at(when)
+.. function:: timeout_at(when)
Similar to :func:`asyncio.timeout`, except *when* is the absolute time
to stop waiting, or ``None``.
@@ -708,9 +767,6 @@ Timeouts
If the wait is cancelled, the future *aw* is also cancelled.
- .. versionchanged:: 3.10
- Removed the *loop* parameter.
-
.. _asyncio_example_waitfor:
Example::
@@ -741,6 +797,9 @@ Timeouts
.. versionchanged:: 3.10
Removed the *loop* parameter.
+ .. versionchanged:: 3.11
+ Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`.
+
Waiting Primitives
==================
@@ -796,6 +855,10 @@ Waiting Primitives
.. versionchanged:: 3.11
Passing coroutine objects to ``wait()`` directly is forbidden.
+ .. versionchanged:: 3.12
+ Added support for generators yielding tasks.
+
+
.. function:: as_completed(aws, *, timeout=None)
Run :ref:`awaitable objects ` in the *aws*
@@ -806,9 +869,6 @@ Waiting Primitives
Raises :exc:`TimeoutError` if the timeout occurs before
all Futures are done.
- .. versionchanged:: 3.10
- Removed the *loop* parameter.
-
Example::
for coro in as_completed(aws):
@@ -822,6 +882,9 @@ Waiting Primitives
Deprecation warning is emitted if not all awaitable objects in the *aws*
iterable are Future-like objects and there is no running event loop.
+ .. versionchanged:: 3.12
+ Added support for generators yielding tasks.
+
Running in Threads
==================
@@ -953,10 +1016,17 @@ Introspection
.. versionadded:: 3.7
+.. function:: iscoroutine(obj)
+
+ Return ``True`` if *obj* is a coroutine object.
+
+ .. versionadded:: 3.4
+
+
Task Object
===========
-.. class:: Task(coro, *, loop=None, name=None)
+.. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False)
A :class:`Future-like ` object that runs a Python
:ref:`coroutine `. Not thread-safe.
@@ -991,9 +1061,17 @@ Task Object
APIs except :meth:`Future.set_result` and
:meth:`Future.set_exception`.
- Tasks support the :mod:`contextvars` module. When a Task
- is created it copies the current context and later runs its
- coroutine in the copied context.
+ An optional keyword-only *context* argument allows specifying a
+ custom :class:`contextvars.Context` for the *coro* to run in.
+ If no *context* is provided, the Task copies the current context
+ and later runs its coroutine in the copied context.
+
+ An optional keyword-only *eager_start* argument allows eagerly starting
+ the execution of the :class:`asyncio.Task` at task creation time.
+ If set to ``True`` and the event loop is running, the task will start
+ executing the coroutine immediately, until the first time the coroutine
+ blocks. If the coroutine returns or raises without blocking, the task
+ will be finished eagerly and will skip scheduling to the event loop.
.. versionchanged:: 3.7
Added support for the :mod:`contextvars` module.
@@ -1005,6 +1083,12 @@ Task Object
Deprecation warning is emitted if *loop* is not specified
and there is no running event loop.
+ .. versionchanged:: 3.11
+ Added the *context* parameter.
+
+ .. versionchanged:: 3.12
+ Added the *eager_start* parameter.
+
.. method:: done()
Return ``True`` if the Task is *done*.
@@ -1089,14 +1173,23 @@ Task Object
The *limit* argument is passed to :meth:`get_stack` directly.
The *file* argument is an I/O stream to which the output
- is written; by default output is written to :data:`sys.stderr`.
+ is written; by default output is written to :data:`sys.stdout`.
.. method:: get_coro()
Return the coroutine object wrapped by the :class:`Task`.
+ .. note::
+
+ This will return ``None`` for Tasks which have already
+ completed eagerly. See the :ref:`Eager Task Factory `.
+
.. versionadded:: 3.8
+ .. versionchanged:: 3.12
+
+ Newly added eager task execution means result may be ``None``.
+
.. method:: get_context()
Return the :class:`contextvars.Context` object
@@ -1139,7 +1232,9 @@ Task Object
Therefore, unlike :meth:`Future.cancel`, :meth:`Task.cancel` does
not guarantee that the Task will be cancelled, although
suppressing cancellation completely is not common and is actively
- discouraged.
+ discouraged. Should the coroutine nevertheless decide to suppress
+ the cancellation, it needs to call :meth:`Task.uncancel` in addition
+ to catching the exception.
.. versionchanged:: 3.9
Added the *msg* parameter.
@@ -1229,6 +1324,10 @@ Task Object
with :meth:`uncancel`. :class:`TaskGroup` context managers use
:func:`uncancel` in a similar fashion.
+ If end-user code is, for some reason, suppresing cancellation by
+ catching :exc:`CancelledError`, it needs to call this method to remove
+ the cancellation state.
+
.. method:: cancelling()
Return the number of pending cancellation requests to this Task, i.e.,
diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst
index b71006e32b2b896..5f33c6813e74c0b 100644
--- a/Doc/library/asyncio.rst
+++ b/Doc/library/asyncio.rst
@@ -46,9 +46,9 @@ Additionally, there are **low-level** APIs for
*library and framework developers* to:
* create and manage :ref:`event loops `, which
- provide asynchronous APIs for :meth:`networking `,
- running :meth:`subprocesses `,
- handling :meth:`OS signals `, etc;
+ provide asynchronous APIs for :ref:`networking `,
+ running :ref:`subprocesses `,
+ handling :ref:`OS signals `, etc;
* implement efficient protocols using
:ref:`transports `;
@@ -56,6 +56,20 @@ Additionally, there are **low-level** APIs for
* :ref:`bridge ` callback-based libraries and code
with async/await syntax.
+.. _asyncio-cli:
+
+You can experiment with an ``asyncio`` concurrent context in the REPL:
+
+.. code-block:: pycon
+
+ $ python -m asyncio
+ asyncio REPL ...
+ Use "await" directly instead of "asyncio.run()".
+ Type "help", "copyright", "credits" or "license" for more information.
+ >>> import asyncio
+ >>> await asyncio.sleep(10, result='hello')
+ 'hello'
+
.. include:: ../includes/wasm-notavail.rst
.. We use the "rubric" directive here to avoid creating
diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst
deleted file mode 100644
index a3a4e90d05277e8..000000000000000
--- a/Doc/library/asyncore.rst
+++ /dev/null
@@ -1,365 +0,0 @@
-:mod:`asyncore` --- Asynchronous socket handler
-===============================================
-
-.. module:: asyncore
- :synopsis: A base class for developing asynchronous socket handling
- services.
- :deprecated:
-
-.. moduleauthor:: Sam Rushing
-.. sectionauthor:: Christopher Petrilli
-.. sectionauthor:: Steve Holden
-.. heavily adapted from original documentation by Sam Rushing
-
-**Source code:** :source:`Lib/asyncore.py`
-
-.. deprecated-removed:: 3.6 3.12
- The :mod:`asyncore` module is deprecated
- (see :pep:`PEP 594 <594#asyncore>` for details).
- Please use :mod:`asyncio` instead.
-
---------------
-
-.. note::
-
- This module exists for backwards compatibility only. For new code we
- recommend using :mod:`asyncio`.
-
-This module provides the basic infrastructure for writing asynchronous socket
-service clients and servers.
-
-.. include:: ../includes/wasm-notavail.rst
-
-There are only two ways to have a program on a single processor do "more than
-one thing at a time." Multi-threaded programming is the simplest and most
-popular way to do it, but there is another very different technique, that lets
-you have nearly all the advantages of multi-threading, without actually using
-multiple threads. It's really only practical if your program is largely I/O
-bound. If your program is processor bound, then pre-emptive scheduled threads
-are probably what you really need. Network servers are rarely processor
-bound, however.
-
-If your operating system supports the :c:func:`select` system call in its I/O
-library (and nearly all do), then you can use it to juggle multiple
-communication channels at once; doing other work while your I/O is taking
-place in the "background." Although this strategy can seem strange and
-complex, especially at first, it is in many ways easier to understand and
-control than multi-threaded programming. The :mod:`asyncore` module solves
-many of the difficult problems for you, making the task of building
-sophisticated high-performance network servers and clients a snap. For
-"conversational" applications and protocols the companion :mod:`asynchat`
-module is invaluable.
-
-The basic idea behind both modules is to create one or more network
-*channels*, instances of class :class:`asyncore.dispatcher` and
-:class:`asynchat.async_chat`. Creating the channels adds them to a global
-map, used by the :func:`loop` function if you do not provide it with your own
-*map*.
-
-Once the initial channel(s) is(are) created, calling the :func:`loop` function
-activates channel service, which continues until the last channel (including
-any that have been added to the map during asynchronous service) is closed.
-
-
-.. function:: loop([timeout[, use_poll[, map[,count]]]])
-
- Enter a polling loop that terminates after count passes or all open
- channels have been closed. All arguments are optional. The *count*
- parameter defaults to ``None``, resulting in the loop terminating only when all
- channels have been closed. The *timeout* argument sets the timeout
- parameter for the appropriate :func:`~select.select` or :func:`~select.poll`
- call, measured in seconds; the default is 30 seconds. The *use_poll*
- parameter, if true, indicates that :func:`~select.poll` should be used in
- preference to :func:`~select.select` (the default is ``False``).
-
- The *map* parameter is a dictionary whose items are the channels to watch.
- As channels are closed they are deleted from their map. If *map* is
- omitted, a global map is used. Channels (instances of
- :class:`asyncore.dispatcher`, :class:`asynchat.async_chat` and subclasses
- thereof) can freely be mixed in the map.
-
-
-.. class:: dispatcher()
-
- The :class:`dispatcher` class is a thin wrapper around a low-level socket
- object. To make it more useful, it has a few methods for event-handling
- which are called from the asynchronous loop. Otherwise, it can be treated
- as a normal non-blocking socket object.
-
- The firing of low-level events at certain times or in certain connection
- states tells the asynchronous loop that certain higher-level events have
- taken place. For example, if we have asked for a socket to connect to
- another host, we know that the connection has been made when the socket
- becomes writable for the first time (at this point you know that you may
- write to it with the expectation of success). The implied higher-level
- events are:
-
- +----------------------+----------------------------------------+
- | Event | Description |
- +======================+========================================+
- | ``handle_connect()`` | Implied by the first read or write |
- | | event |
- +----------------------+----------------------------------------+
- | ``handle_close()`` | Implied by a read event with no data |
- | | available |
- +----------------------+----------------------------------------+
- | ``handle_accepted()``| Implied by a read event on a listening |
- | | socket |
- +----------------------+----------------------------------------+
-
- During asynchronous processing, each mapped channel's :meth:`readable` and
- :meth:`writable` methods are used to determine whether the channel's socket
- should be added to the list of channels :c:func:`select`\ ed or
- :c:func:`poll`\ ed for read and write events.
-
- Thus, the set of channel events is larger than the basic socket events. The
- full set of methods that can be overridden in your subclass follows:
-
-
- .. method:: handle_read()
-
- Called when the asynchronous loop detects that a :meth:`read` call on the
- channel's socket will succeed.
-
-
- .. method:: handle_write()
-
- Called when the asynchronous loop detects that a writable socket can be
- written. Often this method will implement the necessary buffering for
- performance. For example::
-
- def handle_write(self):
- sent = self.send(self.buffer)
- self.buffer = self.buffer[sent:]
-
-
- .. method:: handle_expt()
-
- Called when there is out of band (OOB) data for a socket connection. This
- will almost never happen, as OOB is tenuously supported and rarely used.
-
-
- .. method:: handle_connect()
-
- Called when the active opener's socket actually makes a connection. Might
- send a "welcome" banner, or initiate a protocol negotiation with the
- remote endpoint, for example.
-
-
- .. method:: handle_close()
-
- Called when the socket is closed.
-
-
- .. method:: handle_error()
-
- Called when an exception is raised and not otherwise handled. The default
- version prints a condensed traceback.
-
-
- .. method:: handle_accept()
-
- Called on listening channels (passive openers) when a connection can be
- established with a new remote endpoint that has issued a :meth:`connect`
- call for the local endpoint. Deprecated in version 3.2; use
- :meth:`handle_accepted` instead.
-
- .. deprecated:: 3.2
-
-
- .. method:: handle_accepted(sock, addr)
-
- Called on listening channels (passive openers) when a connection has been
- established with a new remote endpoint that has issued a :meth:`connect`
- call for the local endpoint. *sock* is a *new* socket object usable to
- send and receive data on the connection, and *addr* is the address
- bound to the socket on the other end of the connection.
-
- .. versionadded:: 3.2
-
-
- .. method:: readable()
-
- Called each time around the asynchronous loop to determine whether a
- channel's socket should be added to the list on which read events can
- occur. The default method simply returns ``True``, indicating that by
- default, all channels will be interested in read events.
-
-
- .. method:: writable()
-
- Called each time around the asynchronous loop to determine whether a
- channel's socket should be added to the list on which write events can
- occur. The default method simply returns ``True``, indicating that by
- default, all channels will be interested in write events.
-
-
- In addition, each channel delegates or extends many of the socket methods.
- Most of these are nearly identical to their socket partners.
-
-
- .. method:: create_socket(family=socket.AF_INET, type=socket.SOCK_STREAM)
-
- This is identical to the creation of a normal socket, and will use the
- same options for creation. Refer to the :mod:`socket` documentation for
- information on creating sockets.
-
- .. versionchanged:: 3.3
- *family* and *type* arguments can be omitted.
-
-
- .. method:: connect(address)
-
- As with the normal socket object, *address* is a tuple with the first
- element the host to connect to, and the second the port number.
-
-
- .. method:: send(data)
-
- Send *data* to the remote end-point of the socket.
-
-
- .. method:: recv(buffer_size)
-
- Read at most *buffer_size* bytes from the socket's remote end-point. An
- empty bytes object implies that the channel has been closed from the
- other end.
-
- Note that :meth:`recv` may raise :exc:`BlockingIOError` , even though
- :func:`select.select` or :func:`select.poll` has reported the socket
- ready for reading.
-
-
- .. method:: listen(backlog)
-
- Listen for connections made to the socket. The *backlog* argument
- specifies the maximum number of queued connections and should be at least
- 1; the maximum value is system-dependent (usually 5).
-
-
- .. method:: bind(address)
-
- Bind the socket to *address*. The socket must not already be bound. (The
- format of *address* depends on the address family --- refer to the
- :mod:`socket` documentation for more information.) To mark
- the socket as re-usable (setting the :const:`SO_REUSEADDR` option), call
- the :class:`dispatcher` object's :meth:`set_reuse_addr` method.
-
-
- .. method:: accept()
-
- Accept a connection. The socket must be bound to an address and listening
- for connections. The return value can be either ``None`` or a pair
- ``(conn, address)`` where *conn* is a *new* socket object usable to send
- and receive data on the connection, and *address* is the address bound to
- the socket on the other end of the connection.
- When ``None`` is returned it means the connection didn't take place, in
- which case the server should just ignore this event and keep listening
- for further incoming connections.
-
-
- .. method:: close()
-
- Close the socket. All future operations on the socket object will fail.
- The remote end-point will receive no more data (after queued data is
- flushed). Sockets are automatically closed when they are
- garbage-collected.
-
-
-.. class:: dispatcher_with_send()
-
- A :class:`dispatcher` subclass which adds simple buffered output capability,
- useful for simple clients. For more sophisticated usage use
- :class:`asynchat.async_chat`.
-
-.. class:: file_dispatcher()
-
- A file_dispatcher takes a file descriptor or :term:`file object` along
- with an optional map argument and wraps it for use with the :c:func:`poll`
- or :c:func:`loop` functions. If provided a file object or anything with a
- :c:func:`fileno` method, that method will be called and passed to the
- :class:`file_wrapper` constructor.
-
- .. availability:: Unix.
-
-.. class:: file_wrapper()
-
- A file_wrapper takes an integer file descriptor and calls :func:`os.dup` to
- duplicate the handle so that the original handle may be closed independently
- of the file_wrapper. This class implements sufficient methods to emulate a
- socket for use by the :class:`file_dispatcher` class.
-
- .. availability:: Unix.
-
-
-.. _asyncore-example-1:
-
-asyncore Example basic HTTP client
-----------------------------------
-
-Here is a very basic HTTP client that uses the :class:`dispatcher` class to
-implement its socket handling::
-
- import asyncore
-
- class HTTPClient(asyncore.dispatcher):
-
- def __init__(self, host, path):
- asyncore.dispatcher.__init__(self)
- self.create_socket()
- self.connect( (host, 80) )
- self.buffer = bytes('GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' %
- (path, host), 'ascii')
-
- def handle_connect(self):
- pass
-
- def handle_close(self):
- self.close()
-
- def handle_read(self):
- print(self.recv(8192))
-
- def writable(self):
- return (len(self.buffer) > 0)
-
- def handle_write(self):
- sent = self.send(self.buffer)
- self.buffer = self.buffer[sent:]
-
-
- client = HTTPClient('www.python.org', '/')
- asyncore.loop()
-
-.. _asyncore-example-2:
-
-asyncore Example basic echo server
-----------------------------------
-
-Here is a basic echo server that uses the :class:`dispatcher` class to accept
-connections and dispatches the incoming connections to a handler::
-
- import asyncore
-
- class EchoHandler(asyncore.dispatcher_with_send):
-
- def handle_read(self):
- data = self.recv(8192)
- if data:
- self.send(data)
-
- class EchoServer(asyncore.dispatcher):
-
- def __init__(self, host, port):
- asyncore.dispatcher.__init__(self)
- self.create_socket()
- self.set_reuse_addr()
- self.bind((host, port))
- self.listen(5)
-
- def handle_accepted(self, sock, addr):
- print('Incoming connection from %s' % repr(addr))
- handler = EchoHandler(sock)
-
- server = EchoServer('localhost', 8080)
- asyncore.loop()
diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst
index f7f038107d11fe5..3dbef69580d9b30 100644
--- a/Doc/library/atexit.rst
+++ b/Doc/library/atexit.rst
@@ -20,6 +20,9 @@ at interpreter termination time they will be run in the order ``C``, ``B``,
program is killed by a signal not handled by Python, when a Python fatal
internal error is detected, or when :func:`os._exit` is called.
+**Note:** The effect of registering or unregistering functions from within
+a cleanup function is undefined.
+
.. versionchanged:: 3.7
When used with C-API subinterpreters, registered functions
are local to the interpreter they were registered in.
@@ -45,6 +48,16 @@ internal error is detected, or when :func:`os._exit` is called.
This function returns *func*, which makes it possible to use it as a
decorator.
+ .. warning::
+ Starting new threads or calling :func:`os.fork` from a registered
+ function can lead to race condition between the main Python
+ runtime thread freeing thread states while internal :mod:`threading`
+ routines or the new process try to use that state. This can lead to
+ crashes rather than clean shutdown.
+
+ .. versionchanged:: 3.12
+ Attempts to start a new thread or :func:`os.fork` a new process
+ in a registered function now leads to :exc:`RuntimeError`.
.. function:: unregister(func)
diff --git a/Doc/library/audioop.rst b/Doc/library/audioop.rst
deleted file mode 100644
index 1f96575d08f5b19..000000000000000
--- a/Doc/library/audioop.rst
+++ /dev/null
@@ -1,287 +0,0 @@
-:mod:`audioop` --- Manipulate raw audio data
-============================================
-
-.. module:: audioop
- :synopsis: Manipulate raw audio data.
- :deprecated:
-
-.. deprecated-removed:: 3.11 3.13
- The :mod:`audioop` module is deprecated
- (see :pep:`PEP 594 <594#audioop>` for details).
-
---------------
-
-The :mod:`audioop` module contains some useful operations on sound fragments.
-It operates on sound fragments consisting of signed integer samples 8, 16, 24
-or 32 bits wide, stored in :term:`bytes-like objects `. All scalar items are
-integers, unless specified otherwise.
-
-.. versionchanged:: 3.4
- Support for 24-bit samples was added.
- All functions now accept any :term:`bytes-like object`.
- String input now results in an immediate error.
-
-.. index::
- single: Intel/DVI ADPCM
- single: ADPCM, Intel/DVI
- single: a-LAW
- single: u-LAW
-
-This module provides support for a-LAW, u-LAW and Intel/DVI ADPCM encodings.
-
-.. This para is mostly here to provide an excuse for the index entries...
-
-A few of the more complicated operations only take 16-bit samples, otherwise the
-sample size (in bytes) is always a parameter of the operation.
-
-The module defines the following variables and functions:
-
-
-.. exception:: error
-
- This exception is raised on all errors, such as unknown number of bytes per
- sample, etc.
-
-
-.. function:: add(fragment1, fragment2, width)
-
- Return a fragment which is the addition of the two samples passed as parameters.
- *width* is the sample width in bytes, either ``1``, ``2``, ``3`` or ``4``. Both
- fragments should have the same length. Samples are truncated in case of overflow.
-
-
-.. function:: adpcm2lin(adpcmfragment, width, state)
-
- Decode an Intel/DVI ADPCM coded fragment to a linear fragment. See the
- description of :func:`lin2adpcm` for details on ADPCM coding. Return a tuple
- ``(sample, newstate)`` where the sample has the width specified in *width*.
-
-
-.. function:: alaw2lin(fragment, width)
-
- Convert sound fragments in a-LAW encoding to linearly encoded sound fragments.
- a-LAW encoding always uses 8 bits samples, so *width* refers only to the sample
- width of the output fragment here.
-
-
-.. function:: avg(fragment, width)
-
- Return the average over all samples in the fragment.
-
-
-.. function:: avgpp(fragment, width)
-
- Return the average peak-peak value over all samples in the fragment. No
- filtering is done, so the usefulness of this routine is questionable.
-
-
-.. function:: bias(fragment, width, bias)
-
- Return a fragment that is the original fragment with a bias added to each
- sample. Samples wrap around in case of overflow.
-
-
-.. function:: byteswap(fragment, width)
-
- "Byteswap" all samples in a fragment and returns the modified fragment.
- Converts big-endian samples to little-endian and vice versa.
-
- .. versionadded:: 3.4
-
-
-.. function:: cross(fragment, width)
-
- Return the number of zero crossings in the fragment passed as an argument.
-
-
-.. function:: findfactor(fragment, reference)
-
- Return a factor *F* such that ``rms(add(fragment, mul(reference, -F)))`` is
- minimal, i.e., return the factor with which you should multiply *reference* to
- make it match as well as possible to *fragment*. The fragments should both
- contain 2-byte samples.
-
- The time taken by this routine is proportional to ``len(fragment)``.
-
-
-.. function:: findfit(fragment, reference)
-
- Try to match *reference* as well as possible to a portion of *fragment* (which
- should be the longer fragment). This is (conceptually) done by taking slices
- out of *fragment*, using :func:`findfactor` to compute the best match, and
- minimizing the result. The fragments should both contain 2-byte samples.
- Return a tuple ``(offset, factor)`` where *offset* is the (integer) offset into
- *fragment* where the optimal match started and *factor* is the (floating-point)
- factor as per :func:`findfactor`.
-
-
-.. function:: findmax(fragment, length)
-
- Search *fragment* for a slice of length *length* samples (not bytes!) with
- maximum energy, i.e., return *i* for which ``rms(fragment[i*2:(i+length)*2])``
- is maximal. The fragments should both contain 2-byte samples.
-
- The routine takes time proportional to ``len(fragment)``.
-
-
-.. function:: getsample(fragment, width, index)
-
- Return the value of sample *index* from the fragment.
-
-
-.. function:: lin2adpcm(fragment, width, state)
-
- Convert samples to 4 bit Intel/DVI ADPCM encoding. ADPCM coding is an adaptive
- coding scheme, whereby each 4 bit number is the difference between one sample
- and the next, divided by a (varying) step. The Intel/DVI ADPCM algorithm has
- been selected for use by the IMA, so it may well become a standard.
-
- *state* is a tuple containing the state of the coder. The coder returns a tuple
- ``(adpcmfrag, newstate)``, and the *newstate* should be passed to the next call
- of :func:`lin2adpcm`. In the initial call, ``None`` can be passed as the state.
- *adpcmfrag* is the ADPCM coded fragment packed 2 4-bit values per byte.
-
-
-.. function:: lin2alaw(fragment, width)
-
- Convert samples in the audio fragment to a-LAW encoding and return this as a
- bytes object. a-LAW is an audio encoding format whereby you get a dynamic
- range of about 13 bits using only 8 bit samples. It is used by the Sun audio
- hardware, among others.
-
-
-.. function:: lin2lin(fragment, width, newwidth)
-
- Convert samples between 1-, 2-, 3- and 4-byte formats.
-
- .. note::
-
- In some audio formats, such as .WAV files, 16, 24 and 32 bit samples are
- signed, but 8 bit samples are unsigned. So when converting to 8 bit wide
- samples for these formats, you need to also add 128 to the result::
-
- new_frames = audioop.lin2lin(frames, old_width, 1)
- new_frames = audioop.bias(new_frames, 1, 128)
-
- The same, in reverse, has to be applied when converting from 8 to 16, 24
- or 32 bit width samples.
-
-
-.. function:: lin2ulaw(fragment, width)
-
- Convert samples in the audio fragment to u-LAW encoding and return this as a
- bytes object. u-LAW is an audio encoding format whereby you get a dynamic
- range of about 14 bits using only 8 bit samples. It is used by the Sun audio
- hardware, among others.
-
-
-.. function:: max(fragment, width)
-
- Return the maximum of the *absolute value* of all samples in a fragment.
-
-
-.. function:: maxpp(fragment, width)
-
- Return the maximum peak-peak value in the sound fragment.
-
-
-.. function:: minmax(fragment, width)
-
- Return a tuple consisting of the minimum and maximum values of all samples in
- the sound fragment.
-
-
-.. function:: mul(fragment, width, factor)
-
- Return a fragment that has all samples in the original fragment multiplied by
- the floating-point value *factor*. Samples are truncated in case of overflow.
-
-
-.. function:: ratecv(fragment, width, nchannels, inrate, outrate, state[, weightA[, weightB]])
-
- Convert the frame rate of the input fragment.
-
- *state* is a tuple containing the state of the converter. The converter returns
- a tuple ``(newfragment, newstate)``, and *newstate* should be passed to the next
- call of :func:`ratecv`. The initial call should pass ``None`` as the state.
-
- The *weightA* and *weightB* arguments are parameters for a simple digital filter
- and default to ``1`` and ``0`` respectively.
-
-
-.. function:: reverse(fragment, width)
-
- Reverse the samples in a fragment and returns the modified fragment.
-
-
-.. function:: rms(fragment, width)
-
- Return the root-mean-square of the fragment, i.e. ``sqrt(sum(S_i^2)/n)``.
-
- This is a measure of the power in an audio signal.
-
-
-.. function:: tomono(fragment, width, lfactor, rfactor)
-
- Convert a stereo fragment to a mono fragment. The left channel is multiplied by
- *lfactor* and the right channel by *rfactor* before adding the two channels to
- give a mono signal.
-
-
-.. function:: tostereo(fragment, width, lfactor, rfactor)
-
- Generate a stereo fragment from a mono fragment. Each pair of samples in the
- stereo fragment are computed from the mono sample, whereby left channel samples
- are multiplied by *lfactor* and right channel samples by *rfactor*.
-
-
-.. function:: ulaw2lin(fragment, width)
-
- Convert sound fragments in u-LAW encoding to linearly encoded sound fragments.
- u-LAW encoding always uses 8 bits samples, so *width* refers only to the sample
- width of the output fragment here.
-
-Note that operations such as :func:`.mul` or :func:`.max` make no distinction
-between mono and stereo fragments, i.e. all samples are treated equal. If this
-is a problem the stereo fragment should be split into two mono fragments first
-and recombined later. Here is an example of how to do that::
-
- def mul_stereo(sample, width, lfactor, rfactor):
- lsample = audioop.tomono(sample, width, 1, 0)
- rsample = audioop.tomono(sample, width, 0, 1)
- lsample = audioop.mul(lsample, width, lfactor)
- rsample = audioop.mul(rsample, width, rfactor)
- lsample = audioop.tostereo(lsample, width, 1, 0)
- rsample = audioop.tostereo(rsample, width, 0, 1)
- return audioop.add(lsample, rsample, width)
-
-If you use the ADPCM coder to build network packets and you want your protocol
-to be stateless (i.e. to be able to tolerate packet loss) you should not only
-transmit the data but also the state. Note that you should send the *initial*
-state (the one you passed to :func:`lin2adpcm`) along to the decoder, not the
-final state (as returned by the coder). If you want to use
-:class:`struct.Struct` to store the state in binary you can code the first
-element (the predicted value) in 16 bits and the second (the delta index) in 8.
-
-The ADPCM coders have never been tried against other ADPCM coders, only against
-themselves. It could well be that I misinterpreted the standards in which case
-they will not be interoperable with the respective standards.
-
-The :func:`find\*` routines might look a bit funny at first sight. They are
-primarily meant to do echo cancellation. A reasonably fast way to do this is to
-pick the most energetic piece of the output sample, locate that in the input
-sample and subtract the whole output sample from the input sample::
-
- def echocancel(outputdata, inputdata):
- pos = audioop.findmax(outputdata, 800) # one tenth second
- out_test = outputdata[pos*2:]
- in_test = inputdata[pos*2:]
- ipos, factor = audioop.findfit(in_test, out_test)
- # Optional (for better cancellation):
- # factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)],
- # out_test)
- prefill = '\0'*(pos+ipos)*2
- postfill = '\0'*(len(inputdata)-len(prefill)-len(outputdata))
- outputdata = prefill + audioop.mul(outputdata, 2, -factor) + postfill
- return audioop.add(inputdata, outputdata, 2)
-
diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst
index a02ba739146aaf3..d5b6af8c1928eff 100644
--- a/Doc/library/base64.rst
+++ b/Doc/library/base64.rst
@@ -53,11 +53,13 @@ The modern interface provides:
Encode the :term:`bytes-like object` *s* using Base64 and return the encoded
:class:`bytes`.
- Optional *altchars* must be a :term:`bytes-like object` of at least
- length 2 (additional characters are ignored) which specifies an alternative
- alphabet for the ``+`` and ``/`` characters. This allows an application to e.g.
- generate URL or filesystem safe Base64 strings. The default is ``None``, for
- which the standard Base64 alphabet is used.
+ Optional *altchars* must be a :term:`bytes-like object` of length 2 which
+ specifies an alternative alphabet for the ``+`` and ``/`` characters.
+ This allows an application to e.g. generate URL or filesystem safe Base64
+ strings. The default is ``None``, for which the standard Base64 alphabet is used.
+
+ May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. Raises a
+ :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`.
.. function:: b64decode(s, altchars=None, validate=False)
@@ -65,9 +67,9 @@ The modern interface provides:
Decode the Base64 encoded :term:`bytes-like object` or ASCII string
*s* and return the decoded :class:`bytes`.
- Optional *altchars* must be a :term:`bytes-like object` or ASCII string of
- at least length 2 (additional characters are ignored) which specifies the
- alternative alphabet used instead of the ``+`` and ``/`` characters.
+ Optional *altchars* must be a :term:`bytes-like object` or ASCII string
+ of length 2 which specifies the alternative alphabet used instead of the
+ ``+`` and ``/`` characters.
A :exc:`binascii.Error` exception is raised
if *s* is incorrectly padded.
@@ -80,6 +82,7 @@ The modern interface provides:
For more information about the strict base64 check, see :func:`binascii.a2b_base64`
+ May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2.
.. function:: standard_b64encode(s)
diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst
index 5a0815faa38eac2..39fabb59bb19844 100644
--- a/Doc/library/binascii.rst
+++ b/Doc/library/binascii.rst
@@ -6,14 +6,13 @@
representations.
.. index::
- module: uu
- module: base64
+ pair: module; base64
--------------
The :mod:`binascii` module contains a number of methods to convert between
binary and various ASCII-encoded binary representations. Normally, you will not
-use these functions directly but use wrapper modules like :mod:`uu` or
+use these functions directly but use wrapper modules like
:mod:`base64` instead. The :mod:`binascii` module contains
low-level functions written in C for greater speed that are used by the
higher-level modules.
@@ -58,10 +57,11 @@ The :mod:`binascii` module defines the following functions:
data will raise :exc:`binascii.Error`.
Valid base64:
- * Conforms to :rfc:`3548`.
- * Contains only characters from the base64 alphabet.
- * Contains no excess data after padding (including excess padding, newlines, etc.).
- * Does not start with a padding.
+
+ * Conforms to :rfc:`3548`.
+ * Contains only characters from the base64 alphabet.
+ * Contains no excess data after padding (including excess padding, newlines, etc.).
+ * Does not start with a padding.
.. versionchanged:: 3.11
Added the *strict_mode* parameter.
@@ -179,8 +179,5 @@ The :mod:`binascii` module defines the following functions:
Support for RFC compliant base64-style encoding in base 16, 32, 64,
and 85.
- Module :mod:`uu`
- Support for UU encoding used on Unix.
-
Module :mod:`quopri`
Support for quoted-printable encoding used in MIME email messages.
diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst
index b85564f17866e0b..8022c596f0af97c 100644
--- a/Doc/library/bisect.rst
+++ b/Doc/library/bisect.rst
@@ -24,6 +24,8 @@ method to determine whether a value has been found. Instead, the
functions only call the :meth:`__lt__` method and will return an insertion
point between values in an array.
+.. _bisect functions:
+
The following functions are provided:
@@ -55,7 +57,7 @@ The following functions are provided:
.. function:: bisect_right(a, x, lo=0, hi=len(a), *, key=None)
bisect(a, x, lo=0, hi=len(a), *, key=None)
- Similar to :func:`bisect_left`, but returns an insertion point which comes
+ Similar to :py:func:`~bisect.bisect_left`, but returns an insertion point which comes
after (to the right of) any existing entries of *x* in *a*.
The returned insertion point *ip* partitions the array *a* into two slices
@@ -70,7 +72,7 @@ The following functions are provided:
Insert *x* in *a* in sorted order.
- This function first runs :func:`bisect_left` to locate an insertion point.
+ This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point.
Next, it runs the :meth:`insert` method on *a* to insert *x* at the
appropriate position to maintain sort order.
@@ -87,10 +89,10 @@ The following functions are provided:
.. function:: insort_right(a, x, lo=0, hi=len(a), *, key=None)
insort(a, x, lo=0, hi=len(a), *, key=None)
- Similar to :func:`insort_left`, but inserting *x* in *a* after any existing
+ Similar to :py:func:`~bisect.insort_left`, but inserting *x* in *a* after any existing
entries of *x*.
- This function first runs :func:`bisect_right` to locate an insertion point.
+ This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point.
Next, it runs the :meth:`insert` method on *a* to insert *x* at the
appropriate position to maintain sort order.
@@ -120,7 +122,7 @@ thoughts in mind:
they are used. Consequently, if the search functions are used in a loop,
the key function may be called again and again on the same array elements.
If the key function isn't fast, consider wrapping it with
- :func:`functools.cache` to avoid duplicate computations. Alternatively,
+ :py:func:`functools.cache` to avoid duplicate computations. Alternatively,
consider searching an array of precomputed keys to locate the insertion
point (as shown in the examples section below).
@@ -140,7 +142,7 @@ thoughts in mind:
Searching Sorted Lists
----------------------
-The above :func:`bisect` functions are useful for finding insertion points but
+The above `bisect functions`_ are useful for finding insertion points but
can be tricky or awkward to use for common searching tasks. The following five
functions show how to transform them into the standard lookups for sorted
lists::
@@ -186,8 +188,8 @@ Examples
.. _bisect-example:
-The :func:`bisect` function can be useful for numeric table lookups. This
-example uses :func:`bisect` to look up a letter grade for an exam score (say)
+The :py:func:`~bisect.bisect` function can be useful for numeric table lookups. This
+example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam score (say)
based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is
a 'B', and so on::
@@ -198,8 +200,8 @@ a 'B', and so on::
>>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]]
['F', 'A', 'C', 'C', 'B', 'A', 'A']
-The :func:`bisect` and :func:`insort` functions also work with lists of
-tuples. The *key* argument can serve to extract the field used for ordering
+The :py:func:`~bisect.bisect` and :py:func:`~bisect.insort` functions also work with
+lists of tuples. The *key* argument can serve to extract the field used for ordering
records in a table::
>>> from collections import namedtuple
@@ -210,10 +212,10 @@ records in a table::
>>> Movie = namedtuple('Movie', ('name', 'released', 'director'))
>>> movies = [
- ... Movie('Jaws', 1975, 'Speilberg'),
+ ... Movie('Jaws', 1975, 'Spielberg'),
... Movie('Titanic', 1997, 'Cameron'),
... Movie('The Birds', 1963, 'Hitchcock'),
- ... Movie('Aliens', 1986, 'Scott')
+ ... Movie('Aliens', 1986, 'Cameron')
... ]
>>> # Find the first movie released after 1960
@@ -228,8 +230,8 @@ records in a table::
>>> pprint(movies)
[Movie(name='The Birds', released=1963, director='Hitchcock'),
Movie(name='Love Story', released=1970, director='Hiller'),
- Movie(name='Jaws', released=1975, director='Speilberg'),
- Movie(name='Aliens', released=1986, director='Scott'),
+ Movie(name='Jaws', released=1975, director='Spielberg'),
+ Movie(name='Aliens', released=1986, director='Cameron'),
Movie(name='Titanic', released=1997, director='Cameron')]
If the key function is expensive, it is possible to avoid repeated function
diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst
index ae5a1598f84b44e..6a95a4a6e8d1832 100644
--- a/Doc/library/bz2.rst
+++ b/Doc/library/bz2.rst
@@ -87,10 +87,11 @@ The :mod:`bz2` module contains:
compressed streams.
:class:`BZ2File` provides all of the members specified by the
- :class:`io.BufferedIOBase`, except for :meth:`detach` and :meth:`truncate`.
+ :class:`io.BufferedIOBase`, except for :meth:`~io.BufferedIOBase.detach`
+ and :meth:`~io.IOBase.truncate`.
Iteration and the :keyword:`with` statement are supported.
- :class:`BZ2File` also provides the following method:
+ :class:`BZ2File` also provides the following methods:
.. method:: peek([n])
@@ -105,14 +106,52 @@ The :mod:`bz2` module contains:
.. versionadded:: 3.3
+ .. method:: fileno()
+
+ Return the file descriptor for the underlying file.
+
+ .. versionadded:: 3.3
+
+ .. method:: readable()
+
+ Return whether the file was opened for reading.
+
+ .. versionadded:: 3.3
+
+ .. method:: seekable()
+
+ Return whether the file supports seeking.
+
+ .. versionadded:: 3.3
+
+ .. method:: writable()
+
+ Return whether the file was opened for writing.
+
+ .. versionadded:: 3.3
+
+ .. method:: read1(size=-1)
+
+ Read up to *size* uncompressed bytes, while trying to avoid
+ making multiple reads from the underlying stream. Reads up to a
+ buffer's worth of data if size is negative.
+
+ Returns ``b''`` if the file is at EOF.
+
+ .. versionadded:: 3.3
+
+ .. method:: readinto(b)
+
+ Read bytes into *b*.
+
+ Returns the number of bytes read (0 for EOF).
+
+ .. versionadded:: 3.3
+
.. versionchanged:: 3.1
Support for the :keyword:`with` statement was added.
- .. versionchanged:: 3.3
- The :meth:`fileno`, :meth:`readable`, :meth:`seekable`, :meth:`writable`,
- :meth:`read1` and :meth:`readinto` methods were added.
-
.. versionchanged:: 3.3
Support was added for *filename* being a :term:`file object` instead of an
actual filename.
@@ -320,9 +359,11 @@ Writing and reading a bzip2-compressed file in binary mode:
>>> with bz2.open("myfile.bz2", "wb") as f:
... # Write compressed data to file
... unused = f.write(data)
+ ...
>>> with bz2.open("myfile.bz2", "rb") as f:
... # Decompress data from file
... content = f.read()
+ ...
>>> content == data # Check equality to original object after round-trip
True
diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst
index 66f59f0e2ced275..157a7537f97dc60 100644
--- a/Doc/library/calendar.rst
+++ b/Doc/library/calendar.rst
@@ -394,6 +394,29 @@ The :mod:`calendar` module exports the following data attributes:
An array that represents the abbreviated days of the week in the current locale.
+.. data:: MONDAY
+ TUESDAY
+ WEDNESDAY
+ THURSDAY
+ FRIDAY
+ SATURDAY
+ SUNDAY
+
+ Aliases for the days of the week,
+ where ``MONDAY`` is ``0`` and ``SUNDAY`` is ``6``.
+
+ .. versionadded:: 3.12
+
+
+.. class:: Day
+
+ Enumeration defining days of the week as integer constants.
+ The members of this enumeration are exported to the module scope as
+ :data:`MONDAY` through :data:`SUNDAY`.
+
+ .. versionadded:: 3.12
+
+
.. data:: month_name
An array that represents the months of the year in the current locale. This
@@ -407,15 +430,56 @@ The :mod:`calendar` module exports the following data attributes:
locale. This follows normal convention of January being month number 1, so it
has a length of 13 and ``month_abbr[0]`` is the empty string.
-.. data:: MONDAY
- TUESDAY
- WEDNESDAY
- THURSDAY
- FRIDAY
- SATURDAY
- SUNDAY
- Aliases for day numbers, where ``MONDAY`` is ``0`` and ``SUNDAY`` is ``6``.
+.. data:: JANUARY
+ FEBRUARY
+ MARCH
+ APRIL
+ MAY
+ JUNE
+ JULY
+ AUGUST
+ SEPTEMBER
+ OCTOBER
+ NOVEMBER
+ DECEMBER
+
+ Aliases for the months of the year,
+ where ``JANUARY`` is ``1`` and ``DECEMBER`` is ``12``.
+
+ .. versionadded:: 3.12
+
+
+.. class:: Month
+
+ Enumeration defining months of the year as integer constants.
+ The members of this enumeration are exported to the module scope as
+ :data:`JANUARY` through :data:`DECEMBER`.
+
+ .. versionadded:: 3.12
+
+
+The :mod:`calendar` module defines the following exceptions:
+
+.. exception:: IllegalMonthError(month)
+
+ A subclass of :exc:`ValueError`,
+ raised when the given month number is outside of the range 1-12 (inclusive).
+
+ .. attribute:: month
+
+ The invalid month number.
+
+
+.. exception:: IllegalWeekdayError(weekday)
+
+ A subclass of :exc:`ValueError`,
+ raised when the given weekday number is outside of the range 0-6 (inclusive).
+
+ .. attribute:: weekday
+
+ The invalid weekday number.
+
.. seealso::
@@ -425,3 +489,146 @@ The :mod:`calendar` module exports the following data attributes:
Module :mod:`time`
Low-level time related functions.
+
+
+.. _calendar-cli:
+
+Command-Line Usage
+------------------
+
+.. versionadded:: 2.5
+
+The :mod:`calendar` module can be executed as a script from the command line
+to interactively print a calendar.
+
+.. code-block:: shell
+
+ python -m calendar [-h] [-L LOCALE] [-e ENCODING] [-t {text,html}]
+ [-w WIDTH] [-l LINES] [-s SPACING] [-m MONTHS] [-c CSS]
+ [year] [month]
+
+
+For example, to print a calendar for the year 2000:
+
+.. code-block:: console
+
+ $ python -m calendar 2000
+ 2000
+
+ January February March
+ Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
+ 1 2 1 2 3 4 5 6 1 2 3 4 5
+ 3 4 5 6 7 8 9 7 8 9 10 11 12 13 6 7 8 9 10 11 12
+ 10 11 12 13 14 15 16 14 15 16 17 18 19 20 13 14 15 16 17 18 19
+ 17 18 19 20 21 22 23 21 22 23 24 25 26 27 20 21 22 23 24 25 26
+ 24 25 26 27 28 29 30 28 29 27 28 29 30 31
+ 31
+
+ April May June
+ Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
+ 1 2 1 2 3 4 5 6 7 1 2 3 4
+ 3 4 5 6 7 8 9 8 9 10 11 12 13 14 5 6 7 8 9 10 11
+ 10 11 12 13 14 15 16 15 16 17 18 19 20 21 12 13 14 15 16 17 18
+ 17 18 19 20 21 22 23 22 23 24 25 26 27 28 19 20 21 22 23 24 25
+ 24 25 26 27 28 29 30 29 30 31 26 27 28 29 30
+
+ July August September
+ Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
+ 1 2 1 2 3 4 5 6 1 2 3
+ 3 4 5 6 7 8 9 7 8 9 10 11 12 13 4 5 6 7 8 9 10
+ 10 11 12 13 14 15 16 14 15 16 17 18 19 20 11 12 13 14 15 16 17
+ 17 18 19 20 21 22 23 21 22 23 24 25 26 27 18 19 20 21 22 23 24
+ 24 25 26 27 28 29 30 28 29 30 31 25 26 27 28 29 30
+ 31
+
+ October November December
+ Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
+ 1 1 2 3 4 5 1 2 3
+ 2 3 4 5 6 7 8 6 7 8 9 10 11 12 4 5 6 7 8 9 10
+ 9 10 11 12 13 14 15 13 14 15 16 17 18 19 11 12 13 14 15 16 17
+ 16 17 18 19 20 21 22 20 21 22 23 24 25 26 18 19 20 21 22 23 24
+ 23 24 25 26 27 28 29 27 28 29 30 25 26 27 28 29 30 31
+ 30 31
+
+
+The following options are accepted:
+
+.. program:: calendar
+
+
+.. option:: --help, -h
+
+ Show the help message and exit.
+
+
+.. option:: --locale LOCALE, -L LOCALE
+
+ The locale to use for month and weekday names.
+ Defaults to English.
+
+
+.. option:: --encoding ENCODING, -e ENCODING
+
+ The encoding to use for output.
+ :option:`--encoding` is required if :option:`--locale` is set.
+
+
+.. option:: --type {text,html}, -t {text,html}
+
+ Print the calendar to the terminal as text,
+ or as an HTML document.
+
+
+.. option:: year
+
+ The year to print the calendar for.
+ Must be a number between 1 and 9999.
+ Defaults to the current year.
+
+
+.. option:: month
+
+ The month of the specified :option:`year` to print the calendar for.
+ Must be a number between 1 and 12,
+ and may only be used in text mode.
+ Defaults to printing a calendar for the full year.
+
+
+*Text-mode options:*
+
+.. option:: --width WIDTH, -w WIDTH
+
+ The width of the date column in terminal columns.
+ The date is printed centred in the column.
+ Any value lower than 2 is ignored.
+ Defaults to 2.
+
+
+.. option:: --lines LINES, -l LINES
+
+ The number of lines for each week in terminal rows.
+ The date is printed top-aligned.
+ Any value lower than 1 is ignored.
+ Defaults to 1.
+
+
+.. option:: --spacing SPACING, -s SPACING
+
+ The space between months in columns.
+ Any value lower than 2 is ignored.
+ Defaults to 6.
+
+
+.. option:: --months MONTHS, -m MONTHS
+
+ The number of months printed per row.
+ Defaults to 3.
+
+
+*HTML-mode options:*
+
+.. option:: --css CSS, -c CSS
+
+ The path of a CSS stylesheet to use for the calendar.
+ This must either be relative to the generated HTML,
+ or an absolute HTTP or ``file:///`` URL.
diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst
deleted file mode 100644
index 295a601a7bf197f..000000000000000
--- a/Doc/library/cgi.rst
+++ /dev/null
@@ -1,564 +0,0 @@
-:mod:`cgi` --- Common Gateway Interface support
-===============================================
-
-.. module:: cgi
- :synopsis: Helpers for running Python scripts via the Common Gateway Interface.
- :deprecated:
-
-**Source code:** :source:`Lib/cgi.py`
-
-.. index::
- pair: WWW; server
- pair: CGI; protocol
- pair: HTTP; protocol
- pair: MIME; headers
- single: URL
- single: Common Gateway Interface
-
-.. deprecated-removed:: 3.11 3.13
- The :mod:`cgi` module is deprecated
- (see :pep:`PEP 594 <594#cgi>` for details and alternatives).
-
- The :class:`FieldStorage` class can typically be replaced with
- :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests,
- and the :mod:`email.message` module or
- `multipart `_ for ``POST`` and ``PUT``.
- Most :ref:`utility functions ` have replacements.
-
---------------
-
-Support module for Common Gateway Interface (CGI) scripts.
-
-This module defines a number of utilities for use by CGI scripts written in
-Python.
-
-The global variable ``maxlen`` can be set to an integer indicating the maximum
-size of a POST request. POST requests larger than this size will result in a
-:exc:`ValueError` being raised during parsing. The default value of this
-variable is ``0``, meaning the request size is unlimited.
-
-.. include:: ../includes/wasm-notavail.rst
-
-Introduction
-------------
-
-.. _cgi-intro:
-
-A CGI script is invoked by an HTTP server, usually to process user input
-submitted through an HTML ``