ARROW-7852: [Python] 0.16.0 wheels not compatible with older numpy

Closes apache#6767 from kszucs/ARROW-7852 Authored-by: Krisztián Szűcs <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
ClickHouse · Apr 2, 2020 · 6a583e5 · 6a583e5
1 parent fad2165
commit 6a583e5
Show file tree

Hide file tree

Showing 19 changed files with 43 additions and 46 deletions.
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -184,9 +184,8 @@ jobs:
         run: |
           brew bundle --file=cpp/Brewfile
           brew install coreutils python
-          pip3 install -r python/requirements.txt \
-                       -r python/requirements-test.txt \
-                       cython
+          pip3 install -r python/requirements-build.txt \
+                       -r python/requirements-test.txt
       - name: Build
         shell: bash
         run: |

diff --git a/ci/docker/linux-apt-python-3.dockerfile b/ci/docker/linux-apt-python-3.dockerfile
@@ -29,12 +29,12 @@ RUN apt-get update -y -q && \
 RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
     ln -s /usr/bin/pip3 /usr/local/bin/pip
 
-COPY python/requirements.txt \
+COPY python/requirements-build.txt \
      python/requirements-test.txt \
      /arrow/python/
 
 RUN pip install \
-    -r arrow/python/requirements.txt \
+    -r arrow/python/requirements-build.txt \
     -r arrow/python/requirements-test.txt \
     setuptools
 

diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile
@@ -71,14 +71,8 @@ RUN /arrow/ci/scripts/r_deps.sh /arrow
 RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
     ln -s /usr/bin/pip3 /usr/local/bin/pip
 
-COPY python/requirements.txt \
-     python/requirements-test.txt \
-     /arrow/python/
-
-RUN pip install \
-    -r arrow/python/requirements.txt \
-    cython \
-    setuptools
+COPY python/requirements-build.txt /arrow/python/
+RUN pip install -r arrow/python/requirements-build.txt
 
 ENV \
     ARROW_BUILD_STATIC=OFF \

diff --git a/ci/docker/linux-dnf-python-3.dockerfile b/ci/docker/linux-dnf-python-3.dockerfile
@@ -26,12 +26,12 @@ RUN dnf install -y \
 RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
     ln -s /usr/bin/pip3 /usr/local/bin/pip
 
-COPY python/requirements.txt \
+COPY python/requirements-build.txt \
      python/requirements-test.txt \
      /arrow/python/
 
 RUN pip install \
-    -r arrow/python/requirements.txt \
+    -r arrow/python/requirements-build.txt \
     -r arrow/python/requirements-test.txt
 
 ENV ARROW_PYTHON=ON \

diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
@@ -168,10 +168,7 @@ python/MANIFEST.in
 python/manylinux1/.dockerignore
 python/pyarrow/includes/__init__.pxd
 python/pyarrow/tests/__init__.py
-python/requirements.txt
-python/requirements-test.txt
-python/requirements-wheel.txt
-python/requirements-build.txt
+python/requirements*.txt
 pax_global_header
 MANIFEST.in
 __init__.pxd

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
@@ -363,7 +363,7 @@ test_csharp() {
 test_python() {
   pushd python
 
-  pip install -r requirements.txt -r requirements-test.txt
+  pip install -r requirements-build.txt -r requirements-test.txt
 
   export PYARROW_WITH_DATASET=1
   export PYARROW_WITH_GANDIVA=1

diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml
@@ -20,7 +20,6 @@ os: Visual Studio 2015
 environment:
   ARCH: "64"
   GENERATOR: Visual Studio 14 2015 Win64
-  NUMPY_VERSION: "1.14"
   PYTHON_VERSION: "{{ python_version }}"
   ARROW_SRC: C:\apache-arrow
   PYARROW_VERSION: {{ arrow.no_rc_version }}

diff --git a/dev/tasks/python-wheels/manylinux-test.sh b/dev/tasks/python-wheels/manylinux-test.sh
@@ -25,7 +25,7 @@ python --version
 # Install built wheel
 pip install -q /arrow/python/$WHEEL_DIR/dist/*.whl
 # Install test dependencies (pip won't work after removing system zlib)
-pip install -q -r /arrow/python/requirements-test.txt
+pip install -q -r /arrow/python/requirements-wheel-test.txt
 # Run pyarrow tests
 pytest -rs --pyargs pyarrow
 

diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh
@@ -50,7 +50,7 @@ function build_wheel {
     export ARROW_HOME=`pwd`/arrow-dist
     export PARQUET_HOME=`pwd`/arrow-dist
 
-    pip install $(pip_opts) -r python/requirements-wheel.txt cython
+    pip install $(pip_opts) -r python/requirements-wheel-build.txt
 
     export PYARROW_WITH_GANDIVA=1
     export BUILD_ARROW_GANDIVA=ON
@@ -138,7 +138,7 @@ function run_unit_tests {
     pushd $1
 
     # Install test dependencies
-    pip install $(pip_opts) -r python/requirements-test.txt
+    pip install $(pip_opts) -r python/requirements-wheel-test.txt
 
     # Run pyarrow tests
     pytest -rs --pyargs pyarrow

diff --git a/dev/tasks/python-wheels/win-build-3.5.bat b/dev/tasks/python-wheels/win-build-3.5.bat
@@ -28,7 +28,7 @@ call conda.bat activate wheel-build
 
 @rem Cannot use conda_env_python.yml here because conda-forge has
 @rem ceased providing up-to-date packages for Python 3.5
-pip install -r %ARROW_SRC%\python\requirements-wheel.txt
+pip install -r %ARROW_SRC%\python\requirements-wheel-build.txt
 
 set ARROW_HOME=%CONDA_PREFIX%\Library
 set PARQUET_HOME=%CONDA_PREFIX%\Library
@@ -79,6 +79,7 @@ call conda.bat activate wheel-test
 
 @rem install the built wheel
 pip install -vv %ARROW_SRC%\python\dist\pyarrow-%PYARROW_VERSION%-cp35-cp35m-win_amd64.whl || exit /B
+pip install -q -r %ARROW_SRC%\python\requirements-wheel-test.txt || exit /B
 
 @rem test the imports
 python -c "import pyarrow; import pyarrow.parquet" || exit /B

diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat
@@ -23,15 +23,13 @@ conda update --yes --quiet conda
 conda create -n wheel-build -q -y -c conda-forge ^
     --file=%ARROW_SRC%\ci\conda_env_cpp.yml ^
     --file=%ARROW_SRC%\ci\conda_env_gandiva.yml ^
-    python=%PYTHON_VERSION% ^
-    numpy=%NUMPY_VERSION% ^
-    || exit /B
+    python=%PYTHON_VERSION% || exit /B
 
 call conda.bat activate wheel-build
 
 @rem Cannot use conda_env_python.yml here because conda-forge has
 @rem ceased providing up-to-date packages for Python 3.5
-pip install -r %ARROW_SRC%\python\requirements-wheel.txt
+pip install -r %ARROW_SRC%\python\requirements-wheel-build.txt
 
 set ARROW_HOME=%CONDA_PREFIX%\Library
 set PARQUET_HOME=%CONDA_PREFIX%\Library
@@ -90,14 +88,12 @@ set ARROW_TEST_DATA=%ARROW_SRC%\testing\data
 
 @rem test the wheel
 @rem TODO For maximum reliability, we should test in a plain virtualenv instead.
-conda create -n wheel-test -c conda-forge -q -y ^
-    --file %ARROW_SRC%\ci\conda_env_python.yml ^
-    python=%PYTHON_VERSION% ^
-    numpy=%NUMPY_VERSION% || exit /B
+conda create -n wheel-test -c conda-forge -q -y python=%PYTHON_VERSION% || exit /B
 call conda.bat activate wheel-test
 
 @rem install the built wheel
 pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B
+pip install -q -r %ARROW_SRC%\python\requirements-wheel-test.txt || exit /B
 
 @rem test the imports
 python -c "import pyarrow; import pyarrow.parquet; import pyarrow.flight; import pyarrow.dataset; import pyarrow.gandiva;" || exit /B

diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
@@ -73,7 +73,7 @@ py_libname=$(${PYTHON_INTERPRETER} -c "import sysconfig; print(sysconfig.get_con
 touch ${CPYTHON_PATH}/lib/${py_libname}
 
 echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
-$PIP install -r requirements-wheel.txt
+$PIP install -r requirements-wheel-build.txt
 
 export PYARROW_WITH_DATASET=1
 export PYARROW_WITH_FLIGHT=1

diff --git a/python/manylinux201x/build_arrow.sh b/python/manylinux201x/build_arrow.sh
@@ -72,7 +72,7 @@ py_libname=$(${PYTHON_INTERPRETER} -c "import sysconfig; print(sysconfig.get_con
 touch ${CPYTHON_PATH}/lib/${py_libname}
 
 echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
-$PIP install -r requirements-wheel.txt
+$PIP install -r requirements-wheel-build.txt
 
 export PYARROW_WITH_DATASET=1
 export PYARROW_WITH_FLIGHT=1

diff --git a/python/requirements-build.txt b/python/requirements-build.txt
@@ -1,3 +1,3 @@
--r requirements.txt
--r requirements-test.txt
-cython
+cython>=0.29
+numpy>=1.14
+setuptools_scm
diff --git a/python/requirements-test.txt b/python/requirements-test.txt
@@ -1,5 +1,4 @@
 cffi
-cython
 hypothesis==5.0; python_version <= "3.5.2"
 hypothesis; python_version > "3.5.2"
 pandas==0.24; python_version <= "3.5.2"

diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt
@@ -0,0 +1,7 @@
+Keras-Preprocessing
+cython>=0.29.11
+numpy==1.14.5; python_version < "3.8"
+numpy==1.17.3; python_version >= "3.8"
+pandas
+setuptools_scm==3.2.0
+wheel
diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt
@@ -0,0 +1,12 @@
+cffi
+cython
+hypothesis==5.0; python_version <= "3.5.2"
+hypothesis; python_version > "3.5.2"
+numpy==1.14.5; python_version < "3.8"
+numpy==1.17.3; python_version >= "3.8"
+pandas==0.24; python_version <= "3.5.2"
+pandas; python_version > "3.5.2"
+pickle5; python_version == "3.6" or python_version == "3.7"
+pytest
+pytest-lazy-fixture
+pytz
diff --git a/python/requirements-wheel.txt b/python/requirements-wheel.txt
diff --git a/python/requirements.txt b/python/requirements.txt