Use GitHub variables to reduce duplication + push images early

apache · Dec 19, 2024 · 771cf66 · 771cf66
1 parent 55e4cc0
commit 771cf66
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 49 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -40,6 +40,7 @@ repos:
           (
           ?^ci/docker/conda-python-emscripten\.dockerfile$|
           ?^ci/docker/python-wheel-windows-test-vs2019\.dockerfile$|
+          ?^ci/docker/python-wheel-windows-test-vs2019-base\.dockerfile$|
           )
         types: []
   - repo: https://github.com/pycqa/flake8

diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -21,6 +21,7 @@
 # based on mcr.microsoft.com/windows/servercore:ltsc2019
 # contains choco and vs2019 preinstalled
 ARG base
+# hadolint ignore=DL3006
 FROM ${base}
 
 # hadolint shell=cmd.exe

diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat
@@ -51,8 +51,10 @@ if "%PYTHON%"=="3.13t" (
         --pre ^
         --prefer-binary ^
         -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
-    @REM Avoid crash when importing cffi from test_cffi
-    %PYTHON_CMD% -m pip uninstall -y cffi || exit /B 1
+    @REM 1. cffi-based tests would crash when importing cffi.
+    @REM 2. cython-based tests would crash when importing the compiled extension module
+    @REM    (presumably because of https://github.com/pypa/setuptools/issues/4662)
+    %PYTHON_CMD% -m pip uninstall -y cffi cython || exit /B 1
 ) ELSE (
     %PYTHON_CMD% -m pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
 )

diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml
@@ -26,6 +26,7 @@ jobs:
     env:
       # archery uses this environment variable
       PYTHON: "{{ python_version }}"
+      PYTHON_ABI_TAG: "{{ python_abi_tag }}"
       # this is a private repository at the moment (mostly because of licensing
       # consideration of windows images with visual studio), but anyone can
       # recreate the image by manually building it via:
@@ -43,9 +44,17 @@ jobs:
       {{ macros.github_login_ghcr()|indent }}
       {{ macros.github_install_archery()|indent }}
 
+      - name: Prepare
+        shell: bash
+        run: |
+          if [ "${PYTHON_ABI_TAG}" = "cp313t" ]; then
+            test_image_prefix=python-free-threaded
+          else
+            test_image_prefix=python
+          fi
+          echo "TEST_IMAGE_PREFIX=${test_image_prefix}" >> ${GITHUB_ENV}
+
       - name: Build wheel
-        if: |
-          '{{ python_abi_tag }}' != 'cp313t'
         shell: cmd
         run: |
           cd arrow
@@ -56,72 +65,33 @@ jobs:
           @rem We can remove this workaround once we find a way to use
           @rem pulled caches when build an image.
           echo on
-          archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019-base
-          archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019
+          archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-vs2019
           if errorlevel 1 (
-            archery docker build --no-pull python-wheel-windows-vs2019 || exit /B 1
+            archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2019 || exit /B 1
           )
           archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019
 
-      - name: Build free-threaded wheel
-        if: |
-          '{{ python_abi_tag }}' == 'cp313t'
-        shell: cmd
-        run: |
-          cd arrow
-          @rem We want to use only
-          @rem   archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-free-threaded-wheel-windows-vs2019
-          @rem but it doesn't use pulled caches.
-          @rem It always build an image from scratch.
-          @rem We can remove this workaround once we find a way to use
-          @rem pulled caches when build an image.
-          echo on
-          archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2019-base
-          archery docker pull --no-ignore-pull-failures python-free-threaded-wheel-windows-vs2019
-          if errorlevel 1 (
-            archery docker build --no-pull python-free-threaded-wheel-windows-vs2019 || exit /B 1
-          )
-          archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-free-threaded-wheel-windows-vs2019
-
       - uses: actions/upload-artifact@v4
         with:
           name: wheel
           path: arrow/python/dist/*.whl
 
       - name: Test wheel
-        if: |
-          '{{ python_abi_tag }}' != 'cp313t'
         shell: cmd
         run: |
           cd arrow
-          archery docker run python-wheel-windows-test
+          archery docker run %TEST_IMAGE_PREFIX%-wheel-windows-test
 
-      - name: Test free-threaded wheel
-        if: |
-          '{{ python_abi_tag }}' == 'cp313t'
+      - name: Push Docker image
         shell: cmd
         run: |
           cd arrow
-          archery docker run python-free-threaded-wheel-windows-test
+          archery docker push %TEST_IMAGE_PREFIX%-wheel-windows-vs2019
+          archery docker push %TEST_IMAGE_PREFIX%-wheel-windows-test
 
       {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }}
       {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }}
       {{ macros.github_upload_wheel_scientific_python("arrow/python/dist/*.whl")|indent }}
 
       {% if arrow.is_default_branch() %}
-      - name: Push Docker image
-        if: |
-          '{{ python_abi_tag }}' != 'cp313t'
-        shell: cmd
-        run: |
-          cd arrow
-          archery docker push python-wheel-windows-vs2019
-
-      - name: Push free-threaded Docker image
-        if: |
-          '{{ python_abi_tag }}' == 'cp313t'
-        shell: cmd
-        run: |
-          cd arrow
-          archery docker push python-wheel-windows-vs2019
       {% endif %}