chore(ci): Fix failing weekly jobs (#603)

This PR:

- Updates the integration test workflow to not cache a weekly build,
which was failing with "no space left on device". It also runs more
tests than are strictly necessary at the commit level (and takes
slightly longer without the cached weekly build), so this PR makes it
part of a weekly run instead.
- Updates the alpine dockerfile to avoid a `desc` dependency, since this
fails to install on s390x
- Update two tests to account for a function that returns the value of
`errno` (that might not be the same everywhere)

It also updates the weekly jobs for consistency:

- Updates the weekly jobs to always run on the `maint-**` branches
- Update all of them to cancel previous runs of the same workflow
(because many of them have long-running jobs)

It doesn't fix IPC writing on big endian (a different kind of failure
that I'll punt to another PR).

---------

Co-authored-by: Jacob Wujciak-Jens <[email protected]>

Author: paleolimbot

.github/workflows/docker-build.yaml

@@ -46,7 +46,6 @@ jobs:
           - { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64", service: "verify" }
           - { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64", service: "verify" }
           - { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64", service: "verify" }
-          - { runs_on: "ubuntu-latest", arch: "amd64", platform: "integration", service: "integration" }
 
           - { runs_on: ["self-hosted", "arm"], platform: "ubuntu", arch: "arm64", service: "verify" }
           - { runs_on: ["self-hosted", "arm"], platform: "fedora", arch: "arm64", service: "verify" }

.github/workflows/integration.yaml

@@ -21,14 +21,21 @@ on:
   push:
     branches:
       - main
+      - 'maint-*'
   pull_request:
     paths:
       - .github/workflows/integration.yaml
-      - src/nanoarrow/**
+      - docker-compose.yml
+  schedule:
+    - cron: '5 0 * * 0'
 
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   integration:
     runs-on: ubuntu-latest
@@ -37,5 +44,5 @@ jobs:
 
       - name: Run integration tests
         run: |
-          echo "::group::Docker Pull"
+          echo "::group::Docker Build"
           docker compose run --rm -e GITHUB_ACTIONS integration

.github/workflows/python-wheels.yaml

@@ -31,6 +31,7 @@ on:
   push:
     branches:
       - main
+      - 'maint-**'
   workflow_dispatch:
 
 permissions:

.github/workflows/verify.yaml

@@ -18,6 +18,9 @@
 name: Verification
 
 on:
+  push:
+    branches:
+      - 'maint-**'
   workflow_dispatch:
     inputs:
       version:
@@ -43,6 +46,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   verify:
     name: "${{ matrix.config.label }}"

ci/docker/alpine.dockerfile

@@ -33,9 +33,8 @@ ENV NANOARROW_PYTHON_VENV "/venv"
 
 # For R. Note that arrow is not installed (takes too long).
 RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
-RUN R -e 'install.packages("desc", repos = "https://cloud.r-project.org")' && mkdir /tmp/rdeps
 COPY r/DESCRIPTION /tmp/rdeps
-RUN R -e 'install.packages(setdiff(desc::desc("/tmp/rdeps")$get_deps()$package, "arrow"), repos = "https://cloud.r-project.org")'
+RUN R -e 'gsub("\\(.*?\\)", "", read.dcf("/tmp/rdeps")[1, "Suggests"]) |> strsplit("[^A-Za-z0-9.]+") |> unlist(use.names = FALSE) |> setdiff("arrow") |> install.packages(repos = "https://cloud.r-project.org")'
 RUN rm -f ~/.R/Makevars
 
 ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow

docker-compose.yml

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-version: '3.5'
-
 services:
   verify:
     image: ${REPO}:${NANOARROW_PLATFORM}-${NANOARROW_ARCH}
@@ -48,12 +46,10 @@ services:
     command: "/bin/bash /build-docs.sh /nanoarrow"
 
   integration:
-    # Based on an arrow-dev repo image that is amd64 only
-    image: ${REPO}:integration-amd64
+    # Don't cache here (building the image takes about the same amount of time
+    # as downloading on CI)
     build:
       context: .
-      cache_from:
-        - ${REPO}:integration-amd64
       dockerfile: ci/docker/integration.dockerfile
     volumes:
       - ${NANOARROW_DOCKER_SOURCE_DIR}:/arrow-integration/nanoarrow

src/nanoarrow/ipc/reader_test.cc

@@ -101,6 +101,7 @@ TEST(NanoarrowIpcReader, InputStreamBuffer) {
 
 TEST(NanoarrowIpcReader, InputStreamFile) {
   struct ArrowIpcInputStream stream;
+  errno = EINVAL;
   ASSERT_EQ(ArrowIpcInputStreamInitFile(&stream, nullptr, 1), EINVAL);
 
   uint8_t input_data[] = {0x01, 0x02, 0x03, 0x04, 0x05};

src/nanoarrow/ipc/writer_test.cc

@@ -92,6 +92,7 @@ TEST(NanoarrowIpcWriter, OutputStreamFile) {
 
 TEST(NanoarrowIpcWriter, OutputStreamFileError) {
   nanoarrow::ipc::UniqueOutputStream stream;
+  errno = EINVAL;
   EXPECT_EQ(ArrowIpcOutputStreamInitFile(stream.get(), nullptr, /*close_on_release=*/1),
             EINVAL);