From bdb87f2bbe61c2cf739b48cb4af288966e1c4854 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 5 Apr 2024 10:32:54 -0700 Subject: [PATCH 01/26] GH-37929: [Python] begin moving static settings to pyproject.toml --- .github/workflows/dev.yml | 2 +- python/pyproject.toml | 50 +++++++++++++++++++++++++++++++-- python/setup.py | 58 --------------------------------------- 3 files changed, 49 insertions(+), 61 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6111d1d2e5fe3..1ea12b0a4d23d 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | gem install test-unit - pip install "cython>=0.29.31" setuptools six pytest jira + pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm - name: Run Release Test env: ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/python/pyproject.toml b/python/pyproject.toml index 1588e690a7247..57824e6613762 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -24,7 +24,53 @@ requires = [ # continue using oldest-support-numpy. "oldest-supported-numpy>=0.14; python_version<'3.9'", "numpy>=1.25; python_version>='3.9'", - "setuptools_scm", - "setuptools >= 40.1.0", + # configuring setuptools_scm in pyproject.toml requires + # versions released after 2022 + "setuptools_scm[toml]>=8", + "setuptools>=64", "wheel" ] +build-backend = "setuptools.build_meta" + +[project] +name = "pyarrow" +dynamic = ["version"] +requires-python = ">=3.8" +description = "Python library for Apache Arrow" +readme = {file = "README.md", content-type = "text/markdown"} +license = {text = "Apache Software License"} +classifiers = [ + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Langauge :: Python :: 3.12', +] +maintainers = [ + {name = "Apache Arrow Developers", email = "dev@arrow.apache.org"} +] + +[project.urls] +Homepage = "https://arrow.apache.org/" +Documentation = "https://arrow.apache.org/docs/python" +Repository = "https://github.com/apache/arrow" +Issues = "https://github.com/apache/arrow/issues" +Changelog = "https://github.com/apache/arrow/blob/main/CHANGELOG.md" + +[tool.setuptools] +zip-safe=false +include-package-data=true + +[tool.setuptools.packages.find] +where = ["."] + +[tool.setuptools.package-data] +pyarrow = ["*.pxd", "*.pyx", "includes/*.pxd"] + +[tool.setuptools_scm] +root = '..' +version_file = 'pyarrow/_generated_version.py' +version_scheme = 'guess-next-dev' +relative_to = "pyproject.toml" +git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' diff --git a/python/setup.py b/python/setup.py index ed2b7961e5fbb..5dd38e78e7939 100755 --- a/python/setup.py +++ b/python/setup.py @@ -361,31 +361,6 @@ def get_outputs(self): default_version.replace('-SNAPSHOT', 'a0') -# See https://github.com/pypa/setuptools_scm#configuration-parameters -scm_version_write_to_prefix = os.environ.get( - 'SETUPTOOLS_SCM_VERSION_WRITE_TO_PREFIX', setup_dir) - - -def parse_git(root, **kwargs): - """ - Parse function for setuptools_scm that ignores tags for non-C++ - subprojects, e.g. apache-arrow-js-XXX tags. - """ - from setuptools_scm.git import parse - kwargs['describe_command'] =\ - 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' - return parse(root, **kwargs) - - -def guess_next_dev_version(version): - if version.exact: - return version.format_with('{tag}') - else: - def guess_next_version(tag_version): - return default_version.replace('-SNAPSHOT', '') - return version.format_next_version(guess_next_version) - - with open('README.md') as f: long_description = f.read() @@ -420,11 +395,7 @@ def has_ext_modules(foo): setup( - name='pyarrow', packages=packages, - zip_safe=False, - package_data={'pyarrow': ['*.pxd', '*.pyx', 'includes/*.pxd']}, - include_package_data=True, exclude_package_data=exclude_package_data, distclass=BinaryDistribution, # Dummy extension to trigger build_ext @@ -432,35 +403,6 @@ def has_ext_modules(foo): cmdclass={ 'build_ext': build_ext }, - use_scm_version={ - 'root': os.path.dirname(setup_dir), - 'parse': parse_git, - 'write_to': os.path.join(scm_version_write_to_prefix, - 'pyarrow/_generated_version.py'), - 'version_scheme': guess_next_dev_version - }, - setup_requires=['setuptools_scm', 'cython >= 0.29.31'] + setup_requires, - install_requires=install_requires, tests_require=['pytest', 'pandas', 'hypothesis'], - python_requires='>=3.8', - description='Python library for Apache Arrow', - long_description=long_description, - long_description_content_type='text/markdown', - classifiers=[ - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - ], - license='Apache License, Version 2.0', - maintainer='Apache Arrow Developers', - maintainer_email='dev@arrow.apache.org', test_suite='pyarrow.tests', - url='https://arrow.apache.org/', - project_urls={ - 'Documentation': 'https://arrow.apache.org/docs/python', - 'Source': 'https://github.com/apache/arrow', - }, ) From 608cbb26b077b2cffce1bb519e373d83f2a62a2a Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 25 Apr 2024 10:50:49 -0700 Subject: [PATCH 02/26] GH-37929: [Python][CI] ensure setuptools/setuptools_scm are the high enough version for pyproject.toml support --- ci/conda_env_python.txt | 4 ++-- python/requirements-build.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt index 4366e30010389..bf915493de302 100644 --- a/ci/conda_env_python.txt +++ b/ci/conda_env_python.txt @@ -26,5 +26,5 @@ numpy>=1.16.6 pytest pytest-faulthandler s3fs>=2023.10.0 -setuptools -setuptools_scm +setuptools>=64 +setuptools_scm>=8 diff --git a/python/requirements-build.txt b/python/requirements-build.txt index 87dcc148ad161..c150c842a0cc6 100644 --- a/python/requirements-build.txt +++ b/python/requirements-build.txt @@ -1,5 +1,5 @@ cython>=0.29.31 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=1.25; python_version>='3.9' -setuptools_scm -setuptools>=38.6.0 +setuptools_scm>=8 +setuptools>=64 From 63d04aedc8787911792195849cedaf8ad43376c3 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 2 May 2024 13:11:55 -0700 Subject: [PATCH 03/26] Remove unnecessary configuration --- python/pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 57824e6613762..6df23cec64be9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -56,7 +56,6 @@ Homepage = "https://arrow.apache.org/" Documentation = "https://arrow.apache.org/docs/python" Repository = "https://github.com/apache/arrow" Issues = "https://github.com/apache/arrow/issues" -Changelog = "https://github.com/apache/arrow/blob/main/CHANGELOG.md" [tool.setuptools] zip-safe=false @@ -72,5 +71,4 @@ pyarrow = ["*.pxd", "*.pyx", "includes/*.pxd"] root = '..' version_file = 'pyarrow/_generated_version.py' version_scheme = 'guess-next-dev' -relative_to = "pyproject.toml" git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' From 62ce04d7db257daee8fe43e6462e2dca6734ca59 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 17 May 2024 13:16:50 -0700 Subject: [PATCH 04/26] Cleanup and add non-optional test dependencies --- python/pyproject.toml | 8 ++++++++ python/setup.py | 25 ------------------------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 6df23cec64be9..8671b69131e2a 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -57,6 +57,14 @@ Documentation = "https://arrow.apache.org/docs/python" Repository = "https://github.com/apache/arrow" Issues = "https://github.com/apache/arrow/issues" +[project.optional-dependencies] +test = [ + 'pytest', + 'hypothesis', + 'cffi', + 'pytz' +] + [tool.setuptools] zip-safe=false include-package-data=true diff --git a/python/setup.py b/python/setup.py index 5dd38e78e7939..6bf4536be2346 100755 --- a/python/setup.py +++ b/python/setup.py @@ -352,36 +352,11 @@ def get_outputs(self): for name in self.get_names()] -# If the event of not running from a git clone (e.g. from a git archive -# or a Python sdist), see if we can set the version number ourselves -default_version = '17.0.0-SNAPSHOT' -if (not os.path.exists('../.git') and - not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')): - os.environ['SETUPTOOLS_SCM_PRETEND_VERSION'] = \ - default_version.replace('-SNAPSHOT', 'a0') - - -with open('README.md') as f: - long_description = f.read() - - class BinaryDistribution(Distribution): def has_ext_modules(foo): return True -install_requires = ( - 'numpy >= 1.16.6', -) - - -# Only include pytest-runner in setup_requires if we're invoking tests -if {'pytest', 'test', 'ptr'}.intersection(sys.argv): - setup_requires = ['pytest-runner'] -else: - setup_requires = [] - - if strtobool(os.environ.get('PYARROW_INSTALL_TESTS', '1')): packages = find_namespace_packages(include=['pyarrow*']) exclude_package_data = {} From 4259cf7c98b422c7132da6650dedb6a9418b1e91 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 17 May 2024 13:34:26 -0700 Subject: [PATCH 05/26] setup.py sdist is now deprecated --- ci/scripts/python_sdist_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_sdist_build.sh b/ci/scripts/python_sdist_build.sh index f9e9359b6f66a..93cb6a3922183 100755 --- a/ci/scripts/python_sdist_build.sh +++ b/ci/scripts/python_sdist_build.sh @@ -23,5 +23,5 @@ source_dir=${1}/python pushd ${source_dir} export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-} -${PYTHON:-python} setup.py sdist +${PYTHON:-python} -m build --sdist popd From 15c5c7a49b51c2077458953b231fdcb855bebbc3 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 17 May 2024 13:56:15 -0700 Subject: [PATCH 06/26] Update to use python3 -m build --sdist --- dev/release/02-source-test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/02-source-test.rb b/dev/release/02-source-test.rb index 149a2b27ac94a..15c94ac134069 100644 --- a/dev/release/02-source-test.rb +++ b/dev/release/02-source-test.rb @@ -81,7 +81,7 @@ def test_csharp_git_commit_information def test_python_version source Dir.chdir("#{@tag_name}/python") do - sh("python3", "setup.py", "sdist") + sh("python3", "-m", "build", "--sdist") if on_release_branch? pyarrow_source_archive = "dist/pyarrow-#{@release_version}.tar.gz" else From 327dd38bdb1c3168b59e52aa2ad932879e83efac Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 17 May 2024 14:13:45 -0700 Subject: [PATCH 07/26] Add build as a dependency in CI --- .github/workflows/dev.yml | 2 +- ci/conda_env_archery.txt | 1 + ci/conda_env_crossbow.txt | 1 + ci/conda_env_python.txt | 1 + dev/release/verify-release-candidate.sh | 2 +- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 2 ++ python/pyproject.toml | 1 + python/requirements-build.txt | 1 + python/requirements-wheel-build.txt | 1 + 9 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 1ea12b0a4d23d..cd2655bf4763b 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | gem install test-unit - pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm + pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm build - name: Run Release Test env: ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci/conda_env_archery.txt b/ci/conda_env_archery.txt index ace7a42acb026..ed15cd44e612c 100644 --- a/ci/conda_env_archery.txt +++ b/ci/conda_env_archery.txt @@ -26,6 +26,7 @@ pygit2 pygithub ruamel.yaml setuptools_scm +build toolz # benchmark diff --git a/ci/conda_env_crossbow.txt b/ci/conda_env_crossbow.txt index 347294650ca28..02472ecf10c07 100644 --- a/ci/conda_env_crossbow.txt +++ b/ci/conda_env_crossbow.txt @@ -22,4 +22,5 @@ jira pygit2 ruamel.yaml setuptools_scm +build toolz diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt index bf915493de302..1f5b8438dcee6 100644 --- a/ci/conda_env_python.txt +++ b/ci/conda_env_python.txt @@ -28,3 +28,4 @@ pytest-faulthandler s3fs>=2023.10.0 setuptools>=64 setuptools_scm>=8 +build diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 3ed871bd5305b..849bfffa12dcf 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -756,7 +756,7 @@ test_python() { show_header "Build and test Python libraries" # Build and test Python - maybe_setup_virtualenv "cython>=0.29.31" numpy "setuptools_scm<8.0.0" setuptools + maybe_setup_virtualenv "cython>=0.29.31" numpy "setuptools_scm<8.0.0" setuptools build maybe_setup_conda --file ci/conda_env_python.txt if [ "${USE_CONDA}" -gt 0 ]; then diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 10ee9c28f8c78..2460a761f529e 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -241,6 +241,7 @@ outputs: - python - setuptools - setuptools_scm + - build run: - {{ pin_subpackage('libarrow', exact=True) }} - {{ pin_compatible('numpy') }} @@ -323,6 +324,7 @@ outputs: - python - setuptools - setuptools_scm + - build run: - {{ pin_subpackage('pyarrow', exact=True) }} - python diff --git a/python/pyproject.toml b/python/pyproject.toml index 8671b69131e2a..e2bc27403e9c4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -28,6 +28,7 @@ requires = [ # versions released after 2022 "setuptools_scm[toml]>=8", "setuptools>=64", + "build", "wheel" ] build-backend = "setuptools.build_meta" diff --git a/python/requirements-build.txt b/python/requirements-build.txt index c150c842a0cc6..9945a193c7468 100644 --- a/python/requirements-build.txt +++ b/python/requirements-build.txt @@ -2,4 +2,5 @@ cython>=0.29.31 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=1.25; python_version>='3.9' setuptools_scm>=8 +build setuptools>=64 diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index 3ea57b611f9d8..eed7172ba3275 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -3,4 +3,5 @@ oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=2.0.0rc1; python_version>='3.9' setuptools_scm setuptools>=58 +build wheel From 3fccbd25e431d7c84a42dcdd906ce6efb2eca82d Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 17 May 2024 23:01:59 -0700 Subject: [PATCH 08/26] Revert "setup.py sdist is now deprecated" This reverts commit 6d034579a0f0b5852916997daff7f5de6786038f. --- ci/scripts/python_sdist_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_sdist_build.sh b/ci/scripts/python_sdist_build.sh index 93cb6a3922183..f9e9359b6f66a 100755 --- a/ci/scripts/python_sdist_build.sh +++ b/ci/scripts/python_sdist_build.sh @@ -23,5 +23,5 @@ source_dir=${1}/python pushd ${source_dir} export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-} -${PYTHON:-python} -m build --sdist +${PYTHON:-python} setup.py sdist popd From 2b5a0635be512b325d61cb653ba0978434377c1f Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 17 May 2024 23:02:08 -0700 Subject: [PATCH 09/26] Revert "Update to use python3 -m build --sdist" This reverts commit 3d2d57e550c43a3e81dfbb28a02c2b7946069d25. --- dev/release/02-source-test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/02-source-test.rb b/dev/release/02-source-test.rb index 15c94ac134069..149a2b27ac94a 100644 --- a/dev/release/02-source-test.rb +++ b/dev/release/02-source-test.rb @@ -81,7 +81,7 @@ def test_csharp_git_commit_information def test_python_version source Dir.chdir("#{@tag_name}/python") do - sh("python3", "-m", "build", "--sdist") + sh("python3", "setup.py", "sdist") if on_release_branch? pyarrow_source_archive = "dist/pyarrow-#{@release_version}.tar.gz" else From 9d6eaf258b367991e68ca89d53a5d8e4bb86728e Mon Sep 17 00:00:00 2001 From: anjakefala Date: Fri, 17 May 2024 23:02:15 -0700 Subject: [PATCH 10/26] Revert "Add build as a dependency in CI" This reverts commit cd5df6d1bd8e51e701c2f28f0a83d67ea81265b4. --- .github/workflows/dev.yml | 2 +- ci/conda_env_archery.txt | 1 - ci/conda_env_crossbow.txt | 1 - ci/conda_env_python.txt | 1 - dev/release/verify-release-candidate.sh | 2 +- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 2 -- python/pyproject.toml | 1 - python/requirements-build.txt | 1 - python/requirements-wheel-build.txt | 1 - 9 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index cd2655bf4763b..1ea12b0a4d23d 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | gem install test-unit - pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm build + pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm - name: Run Release Test env: ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci/conda_env_archery.txt b/ci/conda_env_archery.txt index ed15cd44e612c..ace7a42acb026 100644 --- a/ci/conda_env_archery.txt +++ b/ci/conda_env_archery.txt @@ -26,7 +26,6 @@ pygit2 pygithub ruamel.yaml setuptools_scm -build toolz # benchmark diff --git a/ci/conda_env_crossbow.txt b/ci/conda_env_crossbow.txt index 02472ecf10c07..347294650ca28 100644 --- a/ci/conda_env_crossbow.txt +++ b/ci/conda_env_crossbow.txt @@ -22,5 +22,4 @@ jira pygit2 ruamel.yaml setuptools_scm -build toolz diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt index 1f5b8438dcee6..bf915493de302 100644 --- a/ci/conda_env_python.txt +++ b/ci/conda_env_python.txt @@ -28,4 +28,3 @@ pytest-faulthandler s3fs>=2023.10.0 setuptools>=64 setuptools_scm>=8 -build diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 849bfffa12dcf..3ed871bd5305b 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -756,7 +756,7 @@ test_python() { show_header "Build and test Python libraries" # Build and test Python - maybe_setup_virtualenv "cython>=0.29.31" numpy "setuptools_scm<8.0.0" setuptools build + maybe_setup_virtualenv "cython>=0.29.31" numpy "setuptools_scm<8.0.0" setuptools maybe_setup_conda --file ci/conda_env_python.txt if [ "${USE_CONDA}" -gt 0 ]; then diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 2460a761f529e..10ee9c28f8c78 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -241,7 +241,6 @@ outputs: - python - setuptools - setuptools_scm - - build run: - {{ pin_subpackage('libarrow', exact=True) }} - {{ pin_compatible('numpy') }} @@ -324,7 +323,6 @@ outputs: - python - setuptools - setuptools_scm - - build run: - {{ pin_subpackage('pyarrow', exact=True) }} - python diff --git a/python/pyproject.toml b/python/pyproject.toml index e2bc27403e9c4..8671b69131e2a 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -28,7 +28,6 @@ requires = [ # versions released after 2022 "setuptools_scm[toml]>=8", "setuptools>=64", - "build", "wheel" ] build-backend = "setuptools.build_meta" diff --git a/python/requirements-build.txt b/python/requirements-build.txt index 9945a193c7468..c150c842a0cc6 100644 --- a/python/requirements-build.txt +++ b/python/requirements-build.txt @@ -2,5 +2,4 @@ cython>=0.29.31 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=1.25; python_version>='3.9' setuptools_scm>=8 -build setuptools>=64 diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index eed7172ba3275..3ea57b611f9d8 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -3,5 +3,4 @@ oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=2.0.0rc1; python_version>='3.9' setuptools_scm setuptools>=58 -build wheel From f5fb1ed9ee00fde0204a4051a84224c37a168728 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 22 May 2024 12:15:48 +0200 Subject: [PATCH 11/26] Update python/pyproject.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Raúl Cumplido --- python/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index 8671b69131e2a..511cc15a4827e 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -80,3 +80,4 @@ root = '..' version_file = 'pyarrow/_generated_version.py' version_scheme = 'guess-next-dev' git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' +fallback_version = '17.0.0a0' From 74d0f29b633fc129d8e54c38c36879a893e1ab44 Mon Sep 17 00:00:00 2001 From: anjakefala Date: Mon, 27 May 2024 15:59:51 -0700 Subject: [PATCH 12/26] Change tests to modify/refer to fallback_version in pyproject.toml --- dev/release/01-prepare-test.rb | 6 +++--- dev/release/post-11-bump-versions-test.rb | 6 +++--- dev/release/utils-prepare.sh | 10 ++++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb index bf6cfede15c81..114db7cfe7337 100644 --- a/dev/release/01-prepare-test.rb +++ b/dev/release/01-prepare-test.rb @@ -254,10 +254,10 @@ def test_version_pre_tag ], }, { - path: "python/setup.py", + path: "python/pyproject.toml", hunks: [ - ["-default_version = '#{@snapshot_version}'", - "+default_version = '#{@release_version}'"], + ["-fallback_version = '#{@snapshot_version}'", + "+fallback_version = '#{@release_version}'"], ], }, { diff --git a/dev/release/post-11-bump-versions-test.rb b/dev/release/post-11-bump-versions-test.rb index 966c723f70adf..c909713543034 100644 --- a/dev/release/post-11-bump-versions-test.rb +++ b/dev/release/post-11-bump-versions-test.rb @@ -179,10 +179,10 @@ def test_version_post_tag ], }, { - path: "python/setup.py", + path: "python/pyproject.toml", hunks: [ - ["-default_version = '#{@snapshot_version}'", - "+default_version = '#{@next_snapshot_version}'"], + ["-fallback_version = '#{@snapshot_version}'", + "+fallback_version = '#{@snapshot_version}a0'"], ], }, { diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index dfe9b052b09fa..c255e728a335b 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -26,10 +26,12 @@ update_versions() { release) local version=${base_version} local r_version=${base_version} + local python_version=${base_version} ;; snapshot) local version=${next_version}-SNAPSHOT local r_version=${base_version}.9000 + local python_version=${next_version}a0 ;; esac local major_version=${version%%.*} @@ -126,10 +128,10 @@ update_versions() { pushd "${ARROW_DIR}/python" sed -i.bak -E -e \ - "s/^default_version = '.+'/default_version = '${version}'/" \ - setup.py - rm -f setup.py.bak - git add setup.py + "s/^fallback_version = '.+'/fallback_version = '${python_version}'/" \ + pyproject.toml + rm -f pyproject.toml.bak + git add pyproject.toml sed -i.bak -E -e \ "s/^set\(PYARROW_VERSION \".+\"\)/set(PYARROW_VERSION \"${version}\")/" \ CMakeLists.txt From 50f8f3ea644c6be8164e7853ed022bfa3f1312cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 May 2024 10:17:34 +0200 Subject: [PATCH 13/26] Update tests for prepare release and bump_versions --- dev/release/01-prepare-test.rb | 2 +- dev/release/post-11-bump-versions-test.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb index 114db7cfe7337..fbd0b2996077c 100644 --- a/dev/release/01-prepare-test.rb +++ b/dev/release/01-prepare-test.rb @@ -256,7 +256,7 @@ def test_version_pre_tag { path: "python/pyproject.toml", hunks: [ - ["-fallback_version = '#{@snapshot_version}'", + ["-fallback_version = '#{@release_version}a0'", "+fallback_version = '#{@release_version}'"], ], }, diff --git a/dev/release/post-11-bump-versions-test.rb b/dev/release/post-11-bump-versions-test.rb index c909713543034..8ad404ef33202 100644 --- a/dev/release/post-11-bump-versions-test.rb +++ b/dev/release/post-11-bump-versions-test.rb @@ -181,8 +181,8 @@ def test_version_post_tag { path: "python/pyproject.toml", hunks: [ - ["-fallback_version = '#{@snapshot_version}'", - "+fallback_version = '#{@snapshot_version}a0'"], + ["-fallback_version = '#{@release_version}a0'", + "+fallback_version = '#{@next_version}a0'"], ], }, { From 83140a3cf263c618a78635003ec29ae8a9740b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 May 2024 10:51:50 +0200 Subject: [PATCH 14/26] Use python setup.py build_ext --inplace instead of setup.py develop on python minimal examples to use newer PEPs functionality --- python/examples/minimal_build/build_conda.sh | 5 +---- python/examples/minimal_build/build_venv.sh | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 72c3a5f9ea2cd..9a3547c9c3968 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -97,9 +97,6 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH} export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -# You can run either "develop" or "build_ext --inplace". Your pick - -# python setup.py build_ext --inplace -python setup.py develop +python setup.py build_ext --inplace py.test pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 3bd641d0e72c9..cb8d6dc4a7067 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -68,10 +68,7 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH} export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -# You can run either "develop" or "build_ext --inplace". Your pick - -# python setup.py build_ext --inplace -python setup.py develop +python setup.py build_ext --inplace pip install -r $ARROW_ROOT/python/requirements-test.txt From d9bd3cefaec9db4527e65c8a7c6c04761ff11397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 May 2024 11:45:44 +0200 Subject: [PATCH 15/26] Use the same command to build than the one used on python_build.sh --- python/examples/minimal_build/build_conda.sh | 3 ++- python/examples/minimal_build/build_venv.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 9a3547c9c3968..7ebb1fd1ddda7 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -97,6 +97,7 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH} export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -python setup.py build_ext --inplace +# Use the same command that we use on python_build.sh +python -m pip install --no-deps --no-build-isolation -vv . py.test pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index cb8d6dc4a7067..eb881c90c4f38 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -68,7 +68,8 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH} export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -python setup.py build_ext --inplace +# Use the same command that we use on python_build.sh +python -m pip install --no-deps --no-build-isolation -vv . pip install -r $ARROW_ROOT/python/requirements-test.txt From 11e0a32a301931599180797be46e57105f5ed0ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 May 2024 11:57:47 +0200 Subject: [PATCH 16/26] Add missing required numpy to install_requires options --- python/pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index 511cc15a4827e..190b59f30839b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -32,6 +32,10 @@ requires = [ ] build-backend = "setuptools.build_meta" +[options] +install_requires = + numpy >= 1.16.6 + [project] name = "pyarrow" dynamic = ["version"] From 07489afc57bdacc66c9b635bddfc5b31bfbfe78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 May 2024 12:36:37 +0200 Subject: [PATCH 17/26] Add missing quotes and format --- python/pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 190b59f30839b..c06a908056659 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -33,8 +33,9 @@ requires = [ build-backend = "setuptools.build_meta" [options] -install_requires = - numpy >= 1.16.6 +install_requires = [ + "numpy >= 1.16.6" +] [project] name = "pyarrow" From b04b1fa9612fb22ccb1ed4c3e38b8acc0fa893dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 May 2024 16:52:17 +0200 Subject: [PATCH 18/26] Install wheel requirement for minimal env examples, update pytest command, retrieve tags and add some debugging --- dev/tasks/python-minimal-build/github.linux.yml | 2 +- python/examples/minimal_build/Dockerfile.ubuntu | 2 +- python/examples/minimal_build/build_venv.sh | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml index e776312b93f95..d97968b86b362 100644 --- a/dev/tasks/python-minimal-build/github.linux.yml +++ b/dev/tasks/python-minimal-build/github.linux.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest {{ macros.github_set_env(env) }} steps: - {{ macros.github_checkout_arrow(submodules=false)|indent }} + {{ macros.github_checkout_arrow(fetch_depth=0, submodules=false)|indent }} - name: Run minimal build example run: | diff --git a/python/examples/minimal_build/Dockerfile.ubuntu b/python/examples/minimal_build/Dockerfile.ubuntu index ebea4b045e592..148bd7ee3127b 100644 --- a/python/examples/minimal_build/Dockerfile.ubuntu +++ b/python/examples/minimal_build/Dockerfile.ubuntu @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:jammy ENV DEBIAN_FRONTEND=noninteractive diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index eb881c90c4f38..3d4102409b756 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex #---------------------------------------------------------------------- # Change this to whatever makes sense for your system @@ -35,6 +35,7 @@ source $WORKDIR/venv/bin/activate git config --global --add safe.directory $ARROW_ROOT pip install -r $ARROW_ROOT/python/requirements-build.txt +pip install wheel #---------------------------------------------------------------------- # Build C++ library @@ -73,4 +74,6 @@ python -m pip install --no-deps --no-build-isolation -vv . pip install -r $ARROW_ROOT/python/requirements-test.txt -py.test pyarrow +pip freeze +python -c "import pyarrow.lib" +pytest -vv -r s ${PYTEST_ARGS} --pyargs pyarrow From 482eac1497fc9cc7b64624e5cd4ebaa613f5c17e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 May 2024 13:10:02 +0200 Subject: [PATCH 19/26] Add -e to installation --- python/examples/minimal_build/build_conda.sh | 2 +- python/examples/minimal_build/build_venv.sh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 7ebb1fd1ddda7..331e83bb7135f 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -98,6 +98,6 @@ export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja # Use the same command that we use on python_build.sh -python -m pip install --no-deps --no-build-isolation -vv . +python -m pip install --no-deps --no-build-isolation -e . py.test pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 3d4102409b756..6207200ca4fb2 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -70,10 +70,8 @@ export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja # Use the same command that we use on python_build.sh -python -m pip install --no-deps --no-build-isolation -vv . +python -m pip install --no-deps --no-build-isolation -e . pip install -r $ARROW_ROOT/python/requirements-test.txt -pip freeze -python -c "import pyarrow.lib" pytest -vv -r s ${PYTEST_ARGS} --pyargs pyarrow From 3e824750d552683fb35ab73a83a13bc344cbd87d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 May 2024 13:47:34 +0200 Subject: [PATCH 20/26] Revert back to use Ubuntu:focal --- python/examples/minimal_build/Dockerfile.ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/examples/minimal_build/Dockerfile.ubuntu b/python/examples/minimal_build/Dockerfile.ubuntu index 148bd7ee3127b..ebea4b045e592 100644 --- a/python/examples/minimal_build/Dockerfile.ubuntu +++ b/python/examples/minimal_build/Dockerfile.ubuntu @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:jammy +FROM ubuntu:focal ENV DEBIAN_FRONTEND=noninteractive From 2c879c64b740dd4b177330cce3e01a054d5272aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 May 2024 15:05:58 +0200 Subject: [PATCH 21/26] Properly define required numpy dependency on project --- python/pyproject.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index c06a908056659..2673f1de0f780 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -32,15 +32,13 @@ requires = [ ] build-backend = "setuptools.build_meta" -[options] -install_requires = [ - "numpy >= 1.16.6" -] - [project] name = "pyarrow" dynamic = ["version"] requires-python = ">=3.8" +dependencies = [ + "numpy >= 1.16.6" +] description = "Python library for Apache Arrow" readme = {file = "README.md", content-type = "text/markdown"} license = {text = "Apache Software License"} From 2c343e62d2cdd4235924427f76330479c84c1788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 May 2024 17:18:32 +0200 Subject: [PATCH 22/26] Get out of arrow/python directory to not require editable mode installation Change pytest command --- python/examples/minimal_build/Dockerfile.ubuntu | 3 ++- python/examples/minimal_build/build_conda.sh | 5 +++-- python/examples/minimal_build/build_venv.sh | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/python/examples/minimal_build/Dockerfile.ubuntu b/python/examples/minimal_build/Dockerfile.ubuntu index ebea4b045e592..07cd69c082461 100644 --- a/python/examples/minimal_build/Dockerfile.ubuntu +++ b/python/examples/minimal_build/Dockerfile.ubuntu @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:jammy ENV DEBIAN_FRONTEND=noninteractive @@ -32,6 +32,7 @@ RUN apt-get update -y -q && \ python3-dev \ python3-pip \ python3-venv \ + tzdata \ && \ apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 331e83bb7135f..e90c800ae2eb1 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -98,6 +98,7 @@ export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja # Use the same command that we use on python_build.sh -python -m pip install --no-deps --no-build-isolation -e . +python -m pip install --no-deps --no-build-isolation -vv . +popd -py.test pyarrow +pytest -vv -r s ${PYTEST_ARGS} --pyargs pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 6207200ca4fb2..f462c4e9b9d0a 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -70,7 +70,9 @@ export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja # Use the same command that we use on python_build.sh -python -m pip install --no-deps --no-build-isolation -e . +python -m pip install --no-deps --no-build-isolation -vv . + +popd pip install -r $ARROW_ROOT/python/requirements-test.txt From 6f75c2b23fa0564734ef2ad0a56c77eb1be64b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 May 2024 17:54:11 +0200 Subject: [PATCH 23/26] Remove unnecessary test_require and test_suite from setup.py --- python/setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 6bf4536be2346..b738b2f77290e 100755 --- a/python/setup.py +++ b/python/setup.py @@ -378,6 +378,4 @@ def has_ext_modules(foo): cmdclass={ 'build_ext': build_ext }, - tests_require=['pytest', 'pandas', 'hypothesis'], - test_suite='pyarrow.tests', ) From 9adab2662923bb35ab86ad896a78eefa140f945a Mon Sep 17 00:00:00 2001 From: anjakefala Date: Thu, 30 May 2024 10:49:14 -0700 Subject: [PATCH 24/26] Add pandas as a test dependency --- python/pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 2673f1de0f780..2629a98969c2e 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -65,7 +65,8 @@ test = [ 'pytest', 'hypothesis', 'cffi', - 'pytz' + 'pytz', + 'pandas' ] [tool.setuptools] From 50a35c1b1ceed70de674651e044ece237135d9e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 3 Jun 2024 11:48:20 +0200 Subject: [PATCH 25/26] Remove wheel from build-system.requires --- python/examples/minimal_build/build_venv.sh | 1 - python/pyproject.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index f462c4e9b9d0a..47c10d634f7ad 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -35,7 +35,6 @@ source $WORKDIR/venv/bin/activate git config --global --add safe.directory $ARROW_ROOT pip install -r $ARROW_ROOT/python/requirements-build.txt -pip install wheel #---------------------------------------------------------------------- # Build C++ library diff --git a/python/pyproject.toml b/python/pyproject.toml index 2629a98969c2e..f72c3a91eb436 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -28,7 +28,6 @@ requires = [ # versions released after 2022 "setuptools_scm[toml]>=8", "setuptools>=64", - "wheel" ] build-backend = "setuptools.build_meta" From 24054efb984fe33aded930fbb849758821103905 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 4 Jun 2024 14:25:46 +0200 Subject: [PATCH 26/26] add back install wheel --- python/examples/minimal_build/build_venv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 47c10d634f7ad..f462c4e9b9d0a 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -35,6 +35,7 @@ source $WORKDIR/venv/bin/activate git config --global --add safe.directory $ARROW_ROOT pip install -r $ARROW_ROOT/python/requirements-build.txt +pip install wheel #---------------------------------------------------------------------- # Build C++ library