From 9016a836e3b507e9016ce896d5065399a8426435 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 12 Dec 2024 09:44:55 +0100 Subject: [PATCH] GH-44991: [CI][Python] Fix and modernize AppVeyor build (#44999) 1. Use Mambaforge instead of outdated Miniconda install 2. Install PyArrow using pip, not setup.py Also fixes GH-44997. * GitHub Issue: #44991 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- ci/appveyor-cpp-build.bat | 20 +++++++++++--------- ci/appveyor-cpp-setup.bat | 33 ++++++++++++--------------------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 084117f38778a..b1237fc9958c1 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -34,7 +34,6 @@ IF "%ARROW_DEBUG_MEMORY_POOL%"=="" ( set CMAKE_BUILD_PARALLEL_LEVEL=%NUMBER_OF_PROCESSORS% set CTEST_PARALLEL_LEVEL=%NUMBER_OF_PROCESSORS% - call activate arrow @rem The "main" C++ build script for Windows CI @@ -113,12 +112,12 @@ ctest --output-on-failure || exit /B popd +pushd python + @rem @rem Build and install pyarrow @rem -pushd python - set PYARROW_CMAKE_GENERATOR=%GENERATOR% set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS% set PYARROW_PARALLEL=2 @@ -137,6 +136,12 @@ set ARROW_HOME=%CONDA_PREFIX%\Library @rem ARROW-3075; pkgconfig is broken for Parquet for now set PARQUET_HOME=%CONDA_PREFIX%\Library +pip install --no-deps --no-build-isolation -vv --editable . + +@rem +@rem Run pyarrow tests +@rem + @rem Download IANA Timezone Database to a non-standard location to @rem test the configurability of the timezone database path curl https://data.iana.org/time-zones/releases/tzdata2024b.tar.gz --output tzdata.tar.gz || exit /B @@ -150,12 +155,9 @@ rmdir /s /q %USERPROFILE%\Downloads\tzdata @rem (only needed for testing purposes) set PYARROW_TZDATA_PATH=%USERPROFILE%\Downloads\test\tzdata -python setup.py develop -q || exit /B - +set AWS_EC2_METADATA_DISABLED=true set PYTHONDEVMODE=1 -py.test -r sxX --durations=15 --pyargs pyarrow.tests || exit /B +python -m pytest -r sxX --durations=15 pyarrow/tests || exit /B -@rem -@rem Wheels are built and tested separately (see ARROW-5142). -@rem +popd diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index f9463e5074225..912b130acff45 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -17,7 +17,13 @@ @echo on -set "PATH=C:\Miniconda38-x64;C:\Miniconda38-x64\Scripts;C:\Miniconda38-x64\Library\bin;%PATH%" +@rem +@rem The miniconda install on AppVeyor is very outdated, use Mambaforge instead +@rem + +appveyor DownloadFile https://github.com/conda-forge/miniforge/releases/download/24.9.2-0/Mambaforge-Windows-x86_64.exe || exit /B +start /wait "" Mambaforge-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /D=C:\Mambaforge +set "PATH=C:\Mambaforge\scripts;C:\Mambaforge\condabin;%PATH%" @rem @rem Avoid picking up AppVeyor-installed OpenSSL (linker errors with gRPC) @@ -33,26 +39,15 @@ rd /s /q C:\OpenSSL-v30-Win32 rd /s /q C:\OpenSSL-v30-Win64 @rem -@rem Configure miniconda +@rem Configure conda @rem conda config --set auto_update_conda false -conda config --set show_channel_urls True +conda config --set show_channel_urls true +conda config --set always_yes true @rem Help with SSL timeouts to S3 conda config --set remote_connect_timeout_secs 12 -@rem Workaround for ARROW-13636 -conda config --append disallowed_packages pypy3 -conda info -a - -@rem -@rem Install Python to the base environment -@rem -conda install -q -y -c conda-forge python=%PYTHON% || exit /B -@rem Can't use conda-libmamba-solver 2.0.0 -conda config --set solver classic - -@rem Update for newer CA certificates -conda update -q -y -c conda-forge --all || exit /B +conda info -a || exit /B @rem @rem Create conda environment @@ -66,11 +61,8 @@ if "%ARROW_BUILD_GANDIVA%" == "ON" ( ) @rem Install pre-built "toolchain" packages for faster builds set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.txt -@rem Force conda to use conda-forge -conda config --add channels conda-forge -conda config --remove channels defaults @rem Arrow conda environment -conda create -n arrow -y -c conda-forge ^ +conda create -n arrow ^ --file=ci\conda_env_python.txt ^ %CONDA_PACKAGES% ^ "ccache" ^ @@ -97,7 +89,6 @@ if "%ARROW_S3%" == "ON" ( appveyor DownloadFile https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z -FileName C:\Windows\Minio.exe || exit /B ) - @rem @rem Download IANA Timezone Database for unit tests @rem